def _find_and_replace_patterns(content, patterns_and_insertions):
  r"""content: str

  patterns_and_insertions: List[Dict]

  Example for patterns_and_insertions:

      [
          {
              "pattern" :
              r"(?:\\figcompfigures{\s*)(?P<first>.*?)\s*}\s*{\s*(?P<second>.*?)\s*}\s*{\s*(?P<third>.*?)\s*}",
              "insertion" :
              r"\parbox[c]{{{second}\linewidth}}{{\includegraphics[width={third}\linewidth]{{figures/{first}}}}}}",
              "description": "Replace figcompfigures"
          },
      ]
  """
  for pattern_and_insertion in patterns_and_insertions: #  [STATE] pattern_and_insertion = {'pattern': '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}', 'insertion': '\\parbox[c]{{\n            {second}\\linewidth\n        }}{{\n            \\includegraphics[\n                width={third}\\linewidth\n            ]{{\n                figures/{first}\n            }}\n        }} ', 'description': 'Replace figcompfigures'} [/STATE]
    pattern = pattern_and_insertion['pattern'] # [STATE] pattern = '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}' [/STATE]
    insertion = pattern_and_insertion['insertion'] # [STATE] insertion = '\\parbox[c]{{\n            {second}\\linewidth\n        }}{{\n            \\includegraphics[\n                width={third}\\linewidth\n            ]{{\n                figures/{first}\n            }}\n        }} ' [/STATE]
    description = pattern_and_insertion['description'] # [STATE] description = 'Replace figcompfigures' [/STATE]
    logging.info('Processing pattern: %s.', description)
    p = regex.compile(pattern) # [STATE] p = regex.Regex('(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}', flags=regex.V0) [/STATE]
    m = p.search(content) # [STATE] m = <regex.Match object; span=(2, 49), match='\\figcompfigures{\n\timage1.jpg\n}{\n\t\\ww\n}{\n\t1.0\n\t}'> [/STATE]
    while m is not None:
      local_insertion = insertion.format(**m.groupdict()) #  [STATE] local_insertion = '\\parbox[c]{\n            \\ww\\linewidth\n        }{\n            \\includegraphics[\n                width=1.0\\linewidth\n            ]{\n                figures/image1.jpg\n            }\n        } ' [/STATE] [STATE] local_insertion = '\\parbox[c]{\n            \\ww\\linewidth\n        }{\n            \\includegraphics[\n                width=1.0\\linewidth\n            ]{\n                figures/image2.jpg\n            }\n        } ' [/STATE]
      if pattern_and_insertion.get('strip_whitespace', True):
        local_insertion = strip_whitespace(local_insertion) #  [STATE] local_insertion = '\\parbox[c]{\\ww\\linewidth}{\\includegraphics[width=1.0\\linewidth]{figures/image1.jpg}}' [/STATE] [STATE] local_insertion = '\\parbox[c]{\\ww\\linewidth}{\\includegraphics[width=1.0\\linewidth]{figures/image2.jpg}}' [/STATE]
      logging.info(f'Found {content[m.start():m.end()]:<70}')
      logging.info(f'Replacing with {local_insertion:<30}')
      content = content[: m.start()] + local_insertion + content[m.end() :] #  [STATE] content = '& \\parbox[c]{\\ww\\linewidth}{\\includegraphics[width=1.0\\linewidth]{figures/image1.jpg}}\n& \\figcompfigures{image2.jpg}{\\ww}{1.0}' [/STATE] [STATE] content = '& \\parbox[c]{\\ww\\linewidth}{\\includegraphics[width=1.0\\linewidth]{figures/image1.jpg}}\n& \\parbox[c]{\\ww\\linewidth}{\\includegraphics[width=1.0\\linewidth]{figures/image2.jpg}}' [/STATE]
      m = p.search(content) #  [STATE] m = <regex.Match object; span=(89, 126), match='\\figcompfigures{image2.jpg}{\\ww}{1.0}'> [/STATE] [STATE] m = None [/STATE]
    logging.info('Finished pattern: %s.', description)
  return content

_find_and_replace_patterns('& \\figcompfigures{\n\timage1.jpg\n}{\n\t\\ww\n}{\n\t1.0\n\t}\n& \\figcompfigures{image2.jpg}{\\ww}{1.0}', [{'pattern': '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}', 'insertion': '\\parbox[c]{{\n            {second}\\linewidth\n        }}{{\n            \\includegraphics[\n                width={third}\\linewidth\n            ]{{\n                figures/{first}\n            }}\n        }} ', 'description': 'Replace figcompfigures'}])

1: def _find_and_replace_patterns(content, patterns_and_insertions):
2:   r"""content: str
3:
4:   patterns_and_insertions: List[Dict]
5:
6:   Example for patterns_and_insertions:
7:
8:       [
9:           {
10:               "pattern" :
11:               r"(?:\\figcompfigures{\s*)(?P<first>.*?)\s*}\s*{\s*(?P<second>.*?)\s*}\s*{\s*(?P<third>.*?)\s*}",
12:               "insertion" :
13:               r"\parbox[c]{{{second}\linewidth}}{{\includegraphics[width={third}\linewidth]{{figures/{first}}}}}}",
14:               "description": "Replace figcompfigures"
15:           },
16:       ]
17:   """
18:   for pattern_and_insertion in patterns_and_insertions:
19:     pattern = pattern_and_insertion['pattern']
20:     insertion = pattern_and_insertion['insertion']
21:     description = pattern_and_insertion['description']
22:     logging.info('Processing pattern: %s.', description)
23:     p = regex.compile(pattern)
24:     m = p.search(content)
25:     while m is not None:
26:       local_insertion = insertion.format(**m.groupdict())
27:       if pattern_and_insertion.get('strip_whitespace', True):
28:         local_insertion = strip_whitespace(local_insertion)
29:       logging.info(f'Found {content[m.start():m.end()]:<70}')
30:       logging.info(f'Replacing with {local_insertion:<30}')
31:       content = content[: m.start()] + local_insertion + content[m.end() :]
32:       m = p.search(content)
33:     logging.info('Finished pattern: %s.', description)
34:   return content
35:
36: _find_and_replace_patterns('& \\figcompfigures{\n\timage1.jpg\n}{\n\t\\ww\n}{\n\t1.0\n\t}\n& \\figcompfigures{image2.jpg}{\\ww}{1.0}', [{'pattern': '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}', 'insertion': '\\parbox[c]{{\n            {second}\\linewidth\n        }}{{\n            \\includegraphics[\n                width={third}\\linewidth\n            ]{{\n                figures/{first}\n            }}\n        }} ', 'description': 'Replace figcompfigures'}])

Question: What is the value of ' pattern ' in line '19' after '1' th iteration when '_find_and_replace_patterns('& \\figcompfigures{\n\timage1.jpg\n}{\n\t\\ww\n}{\n\t1.0\n\t}\n& \\figcompfigures{image2.jpg}{\\ww}{1.0}', [{'pattern': '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}', 'insertion': '\\parbox[c]{{\n            {second}\\linewidth\n        }}{{\n            \\includegraphics[\n                width={third}\\linewidth\n            ]{{\n                figures/{first}\n            }}\n        }} ', 'description': 'Replace figcompfigures'}])' is executed?

Answer: '(?:\\\\figcompfigures{\\s*)(?P<first>.*?)\\s*}\\s*{\\s*(?P<second>.*?)\\s*}\\s*{\\s*(?P<third>.*?)\\s*}'


def _keep_pattern(haystack, patterns_to_keep):
  """Keeps the strings that match 'patterns_to_keep'."""
  out = [] # [STATE] out = [] [/STATE]
  for item in haystack: #  [STATE] item = 'abc' [/STATE] [STATE] item = 'bca' [/STATE]
    if any((regex.findall(rem, item) for rem in patterns_to_keep)):
      out.append(item) #  [STATE] out = ['abc'] [/STATE] [STATE] out = ['abc', 'bca'] [/STATE]
  return out

_keep_pattern(['abc', 'bca'], ['a'])

1: def _keep_pattern(haystack, patterns_to_keep):
2:   """Keeps the strings that match 'patterns_to_keep'."""
3:   out = []
4:   for item in haystack:
5:     if any((regex.findall(rem, item) for rem in patterns_to_keep)):
6:       out.append(item)
7:   return out
8:
9: _keep_pattern(['abc', 'bca'], ['a'])

Question: What is the value of ' out ' in line '6' after '1' th iteration when '_keep_pattern(['abc', 'bca'], ['a'])' is executed?

Answer: ['abc']


def merge_args_into_config(args, config_params):
  final_args = copy.deepcopy(config_params) # [STATE] final_args = {'input_folder': 'foo_/bar_', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE]
  config_keys = config_params.keys() # [STATE] config_keys = dict_keys(['input_folder', 'resize_images', 'im_size', 'compress_pdf', 'pdf_im_resolution', 'images_allowlist', 'commands_to_delete', 'use_external_tikz']) [/STATE]
  for key, value in args.items(): #  [STATE] key = 'input_folder' [/STATE] [STATE] value = 'foo/bar' [/STATE] [STATE] key = 'resize_images' [/STATE] [STATE] value = False [/STATE] [STATE] key = 'im_size' [/STATE] [STATE] value = 500 [/STATE] [STATE] key = 'compress_pdf' [/STATE] [STATE] key = 'pdf_im_resolution' [/STATE] [STATE] key = 'images_allowlist' [/STATE] [STATE] value = {'path1/': 1000} [/STATE] [STATE] key = 'commands_to_delete' [/STATE] [STATE] value = ['\\todo1'] [/STATE] [STATE] key = 'use_external_tikz' [/STATE] [STATE] value = 'foo/bar/tikz' [/STATE]
    if key in config_keys:
      if any([isinstance(value, t) for t in [str, bool, float, int]]):
        # Overwrites config value with args value.
        final_args[key] = value #  [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE] [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE] [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE] [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE] [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE] [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo/bar/tikz'} [/STATE]
      elif isinstance(value, list):
        # Appends args values to config values.
        final_args[key] = value + config_params[key] # [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE]
      elif isinstance(value, dict):
        # Updates config params with args params.
        final_args[key].update(**value) # [STATE] final_args = {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'} [/STATE]
    else:
      final_args[key] = value
  return final_args

merge_args_into_config({'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path1/': 1000}, 'commands_to_delete': ['\\todo1'], 'use_external_tikz': 'foo/bar/tikz'}, {'input_folder': 'foo_/bar_', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'})

1: def merge_args_into_config(args, config_params):
2:   final_args = copy.deepcopy(config_params)
3:   config_keys = config_params.keys()
4:   for key, value in args.items():
5:     if key in config_keys:
6:       if any([isinstance(value, t) for t in [str, bool, float, int]]):
7:         # Overwrites config value with args value.
8:         final_args[key] = value
9:       elif isinstance(value, list):
10:         # Appends args values to config values.
11:         final_args[key] = value + config_params[key]
12:       elif isinstance(value, dict):
13:         # Updates config params with args params.
14:         final_args[key].update(**value)
15:     else:
16:       final_args[key] = value
17:   return final_args
18:
19: merge_args_into_config({'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path1/': 1000}, 'commands_to_delete': ['\\todo1'], 'use_external_tikz': 'foo/bar/tikz'}, {'input_folder': 'foo_/bar_', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'})

Question: What is the value of ' final_args ' in line '8' after '6' th iteration when 'merge_args_into_config({'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path1/': 1000}, 'commands_to_delete': ['\\todo1'], 'use_external_tikz': 'foo/bar/tikz'}, {'input_folder': 'foo_/bar_', 'resize_images': True, 'im_size': 1000, 'compress_pdf': True, 'pdf_im_resolution': 1000, 'images_allowlist': {'path2/': 1000}, 'commands_to_delete': ['\\todo2'], 'use_external_tikz': 'foo_/bar_/tikz_'})' is executed?

Answer: {'input_folder': 'foo/bar', 'resize_images': False, 'im_size': 500, 'compress_pdf': False, 'pdf_im_resolution': 500, 'images_allowlist': {'path2/': 1000, 'path1/': 1000}, 'commands_to_delete': ['\\todo1', '\\todo2'], 'use_external_tikz': 'foo/bar/tikz'}


def _remove_command(text, command, keep_text=False):
  """Removes '\\command{*}' from the string 'text'.

  Regex `base_pattern` used to match balanced parentheses taken from:
  https://stackoverflow.com/questions/546433/regular-expression-to-match-balanced-parentheses/35271017#35271017
  """
  base_pattern = r'\\' + command + r'\{((?:[^{}]+|\{(?1)\})*)\}' # [STATE] base_pattern = '\\\\todo\\{((?:[^{}]+|\\{(?1)\\})*)\\}' [/STATE]
  # Loops in case of nested commands that need to retain text, e.g.,
  # \red{hello \red{world}}.
  while True:
    all_substitutions = [] # [STATE] all_substitutions = [] [/STATE]
    has_match = False # [STATE] has_match = False [/STATE]
    for match in regex.finditer(base_pattern, text): #  [STATE] match = <regex.Match object; span=(1, 11), match='\\todo{B\nC}'> [/STATE]
      # In case there are only spaces or nothing up to the following newline,
      # adds a percent, not to alter the newlines.
      has_match = True # [STATE] has_match = True [/STATE]
      new_substring = ( # [STATE] new_substring = '' [/STATE]
          ''
          if not keep_text
          else text[match.span()[0] + len(command) + 2 : match.span()[1] - 1]
      )
      if match.span()[1] < len(text):
        next_newline = text[match.span()[1] :].find('\n') # [STATE] next_newline = 1 [/STATE]
        if next_newline != -1:
          text_until_newline = text[ # [STATE] text_until_newline = 'D' [/STATE]
              match.span()[1] : match.span()[1] + next_newline
          ]
          if (
              not text_until_newline or text_until_newline.isspace()
          ) and not keep_text:
            new_substring = '%'
      all_substitutions.append( # [STATE] all_substitutions = [(1, 11, '')] [/STATE]
          (match.span()[0], match.span()[1], new_substring)
      )

    for start, end, new_substring in reversed(all_substitutions): #  [STATE] start = 1 [/STATE] [STATE] end = 11 [/STATE]
      text = text[:start] + new_substring + text[end:] # [STATE] text = 'AD\nE\n\\end{document}' [/STATE]

    if not keep_text or not has_match:
      break

  return text

_remove_command('A\\todo{B\nC}D\nE\n\\end{document}', 'todo', False)

1: def _remove_command(text, command, keep_text=False):
2:   """Removes '\\command{*}' from the string 'text'.
3:
4:   Regex `base_pattern` used to match balanced parentheses taken from:
5:   https://stackoverflow.com/questions/546433/regular-expression-to-match-balanced-parentheses/35271017#35271017
6:   """
7:   base_pattern = r'\\' + command + r'\{((?:[^{}]+|\{(?1)\})*)\}'
8:   # Loops in case of nested commands that need to retain text, e.g.,
9:   # \red{hello \red{world}}.
10:   while True:
11:     all_substitutions = []
12:     has_match = False
13:     for match in regex.finditer(base_pattern, text):
14:       # In case there are only spaces or nothing up to the following newline,
15:       # adds a percent, not to alter the newlines.
16:       has_match = True
17:       new_substring = (
18:           ''
19:           if not keep_text
20:           else text[match.span()[0] + len(command) + 2 : match.span()[1] - 1]
21:       )
22:       if match.span()[1] < len(text):
23:         next_newline = text[match.span()[1] :].find('\n')
24:         if next_newline != -1:
25:           text_until_newline = text[
26:               match.span()[1] : match.span()[1] + next_newline
27:           ]
28:           if (
29:               not text_until_newline or text_until_newline.isspace()
30:           ) and not keep_text:
31:             new_substring = '%'
32:       all_substitutions.append(
33:           (match.span()[0], match.span()[1], new_substring)
34:       )
35:
36:     for start, end, new_substring in reversed(all_substitutions):
37:       text = text[:start] + new_substring + text[end:]
38:
39:     if not keep_text or not has_match:
40:       break
41:
42:   return text
43:
44: _remove_command('A\\todo{B\nC}D\nE\n\\end{document}', 'todo', False)

Question: What is the value of ' has_match ' in line '16' after '1' th iteration when '_remove_command('A\\todo{B\nC}D\nE\n\\end{document}', 'todo', False)' is executed?

Answer: True


def _remove_comments_inline(text):
  """Removes the comments from the string 'text' and ignores % inside \\url{}."""
  if 'auto-ignore' in text:
    return text
  if text.lstrip(' ').lstrip('\t').startswith('%'):
    return ''

  url_pattern = r'\\url\{(?>[^{}]|(?R))*\}' # [STATE] url_pattern = '\\\\url\\{(?>[^{}]|(?R))*\\}' [/STATE]

  def remove_comments(segment): # [STATE] remove_comments = <function _remove_comments_inline.<locals>.remove_comments at 0x7f185616f280> [/STATE]
    """Remove comments from a segment of text."""
    if segment.lstrip().startswith('%'):
      return ''
    match = regex.search(r'(?<!\\)%', segment)
    if match:
      return segment[: match.end()] + '\n'
    else:
      return segment

  # split the text into segments based on \url{} tags
  segments = regex.split(f'({url_pattern})', text) # [STATE] segments = ['Foo %Comment\n'] [/STATE]

  for i in range(len(segments)): #  [STATE] i = 0 [/STATE]
    # only process segments that are not part of a \url{} tag
    if not regex.match(url_pattern, segments[i]):
      segments[i] = remove_comments(segments[i]) # [STATE] segments = ['Foo %\n'] [/STATE]

  final_text = ''.join(segments) # [STATE] final_text = 'Foo %\n' [/STATE]
  return (
      final_text
      if final_text.endswith('\n') or final_text.endswith('\\n')
      else final_text + '\n'
  )

_remove_comments_inline('Foo %Comment\n')

1: def _remove_comments_inline(text):
2:   """Removes the comments from the string 'text' and ignores % inside \\url{}."""
3:   if 'auto-ignore' in text:
4:     return text
5:   if text.lstrip(' ').lstrip('\t').startswith('%'):
6:     return ''
7:
8:   url_pattern = r'\\url\{(?>[^{}]|(?R))*\}'
9:
10:   def remove_comments(segment):
11:     """Remove comments from a segment of text."""
12:     if segment.lstrip().startswith('%'):
13:       return ''
14:     match = regex.search(r'(?<!\\)%', segment)
15:     if match:
16:       return segment[: match.end()] + '\n'
17:     else:
18:       return segment
19:
20:   # split the text into segments based on \url{} tags
21:   segments = regex.split(f'({url_pattern})', text)
22:
23:   for i in range(len(segments)):
24:     # only process segments that are not part of a \url{} tag
25:     if not regex.match(url_pattern, segments[i]):
26:       segments[i] = remove_comments(segments[i])
27:
28:   final_text = ''.join(segments)
29:   return (
30:       final_text
31:       if final_text.endswith('\n') or final_text.endswith('\\n')
32:       else final_text + '\n'
33:   )
34:
35: _remove_comments_inline('Foo %Comment\n')

Question: What is the value of ' segments ' in line '26' after '1' th iteration when '_remove_comments_inline('Foo %Comment\n')' is executed?

Answer: ['Foo %\n']


def _search_reference(filename, contents, strict=False):
  """Returns a match object if filename is referenced in contents, and None otherwise.

  If not strict mode, path prefix and extension are optional.
  """
  if strict:
    # regex pattern for strict=True for path/to/img.ext:
    # \{[\s%]*path/to/img\.ext[\s%]*\}
    filename_regex = filename.replace('.', r'\.')
  else:
    filename_path = Path(filename) # [STATE] filename_path = PosixPath('to/img.ext') [/STATE]

    # make extension optional
    root, extension = filename_path.stem, filename_path.suffix # [STATE] root = 'img' [/STATE] # [STATE] extension = '.ext' [/STATE]
    basename_regex = '{}({})?'.format( # [STATE] basename_regex = 'img(\\.ext)?' [/STATE]
        regex.escape(root), regex.escape(extension)
    )

    # iterate through parent fragments to make path prefix optional
    path_prefix_regex = '' # [STATE] path_prefix_regex = '' [/STATE]
    for fragment in reversed(filename_path.parents): #  [STATE] fragment = PosixPath('.') [/STATE] [STATE] fragment = PosixPath('to') [/STATE]
      if fragment.name == '.':
        continue
      fragment = regex.escape(fragment.name) #  [STATE] fragment = '' [/STATE] [STATE] fragment = 'to' [/STATE]
      path_prefix_regex = '({}{}{})?'.format( #  [STATE] path_prefix_regex = '(/)?' [/STATE] [STATE] path_prefix_regex = '((/)?to/)?' [/STATE]
          path_prefix_regex, fragment, os.sep
      )

    # Regex pattern for strict=True for path/to/img.ext:
    # \{[\s%]*(<path_prefix>)?<basename>(<ext>)?[\s%]*\}
    filename_regex = path_prefix_regex + basename_regex # [STATE] filename_regex = '((/)?to/)?img(\\.ext)?' [/STATE]

  # Some files 'path/to/file' are referenced in tex as './path/to/file' thus
  # adds prefix for relative paths starting with './' or '.\' to regex search.
  filename_regex = r'(.' + os.sep + r')?' + filename_regex # [STATE] filename_regex = '(./)?((/)?to/)?img(\\.ext)?' [/STATE]

  # Pads with braces and optional whitespace/comment characters.
  patn = r'\{{[\s%]*{}[\s%]*\}}'.format(filename_regex) # [STATE] patn = '\\{[\\s%]*(./)?((/)?to/)?img(\\.ext)?[\\s%]*\\}' [/STATE]
  # Picture references in LaTeX are allowed to be in different cases.
  return regex.search(patn, contents, regex.IGNORECASE)

_search_reference('to/img.ext', '{long/path/to/img}', False)

1: def _search_reference(filename, contents, strict=False):
2:   """Returns a match object if filename is referenced in contents, and None otherwise.
3:
4:   If not strict mode, path prefix and extension are optional.
5:   """
6:   if strict:
7:     # regex pattern for strict=True for path/to/img.ext:
8:     # \{[\s%]*path/to/img\.ext[\s%]*\}
9:     filename_regex = filename.replace('.', r'\.')
10:   else:
11:     filename_path = Path(filename)
12:
13:     # make extension optional
14:     root, extension = filename_path.stem, filename_path.suffix
15:     basename_regex = '{}({})?'.format(
16:         regex.escape(root), regex.escape(extension)
17:     )
18:
19:     # iterate through parent fragments to make path prefix optional
20:     path_prefix_regex = ''
21:     for fragment in reversed(filename_path.parents):
22:       if fragment.name == '.':
23:         continue
24:       fragment = regex.escape(fragment.name)
25:       path_prefix_regex = '({}{}{})?'.format(
26:           path_prefix_regex, fragment, os.sep
27:       )
28:
29:     # Regex pattern for strict=True for path/to/img.ext:
30:     # \{[\s%]*(<path_prefix>)?<basename>(<ext>)?[\s%]*\}
31:     filename_regex = path_prefix_regex + basename_regex
32:
33:   # Some files 'path/to/file' are referenced in tex as './path/to/file' thus
34:   # adds prefix for relative paths starting with './' or '.\' to regex search.
35:   filename_regex = r'(.' + os.sep + r')?' + filename_regex
36:
37:   # Pads with braces and optional whitespace/comment characters.
38:   patn = r'\{{[\s%]*{}[\s%]*\}}'.format(filename_regex)
39:   # Picture references in LaTeX are allowed to be in different cases.
40:   return regex.search(patn, contents, regex.IGNORECASE)
41:
42: _search_reference('to/img.ext', '{long/path/to/img}', False)

Question: What is the value of ' fragment ' in line '24' after '2' th iteration when '_search_reference('to/img.ext', '{long/path/to/img}', False)' is executed?

Answer: 'to'


def run_arxiv_cleaner(parameters):
  """Core of the code, runs the actual arXiv cleaner."""

  files_to_delete = [ # [STATE] files_to_delete = ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$'] [/STATE]
      r'\.aux$',
      r'\.sh$',
      r'\.blg$',
      r'\.brf$',
      r'\.log$',
      r'\.out$',
      r'\.ps$',
      r'\.dvi$',
      r'\.synctex.gz$',
      '~$',
      r'\.backup$',
      r'\.gitignore$',
      r'\.DS_Store$',
      r'\.svg$',
      r'^\.idea',
      r'\.dpth$',
      r'\.md5$',
      r'\.dep$',
      r'\.auxlock$',
      r'\.fls$',
      r'\.fdb_latexmk$',
  ]

  if not parameters['keep_bib']:
    files_to_delete.append(r'\.bib$') # [STATE] files_to_delete = ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'] [/STATE]

  parameters.update({ # [STATE] parameters = {'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$']} [/STATE]
      'to_delete': files_to_delete,
      'figures_to_copy_if_referenced': [
          r'\.png$',
          r'\.jpg$',
          r'\.jpeg$',
          r'\.pdf$',
      ],
  })

  logging.info('Collecting file structure.')
  parameters['output_folder'] = _create_out_folder(parameters['input_folder']) # [STATE] parameters = {'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'} [/STATE]

  from_zip = parameters['input_folder'].endswith('.zip') # [STATE] from_zip = False [/STATE]
  tempdir_context = ( # [STATE] tempdir_context = {_exceptions=()} [/STATE]
      tempfile.TemporaryDirectory() if from_zip else contextlib.suppress()
  )

  with tempdir_context as tempdir: # [STATE] tempdir = None [/STATE]

    if from_zip:
      logging.info('Unzipping input folder.')
      shutil.unpack_archive(parameters['input_folder'], tempdir)
      parameters['input_folder'] = tempdir

    splits = _split_all_files(parameters) # [STATE] splits = {'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': []} [/STATE]

    logging.info('Reading all tex files')
    tex_contents = _read_all_tex_contents( # [STATE] tex_contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE]
        splits['tex_in_root'] + splits['tex_not_in_root'], parameters
    )

    for tex_file in tex_contents: #  [STATE] tex_file = 'main.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included_2.tex' [/STATE] [STATE] tex_file = 'figures/figure_included.tikz' [/STATE] [STATE] tex_file = 'figures/figure_included.tex' [/STATE]
      logging.info('Removing comments in file %s.', tex_file)
      tex_contents[tex_file] = _remove_comments_and_commands_to_delete( #  [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE] [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE] [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE] [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE] [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE]
          tex_contents[tex_file], parameters
      )

    for tex_file in tex_contents: #  [STATE] tex_file = 'main.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included_2.tex' [/STATE] [STATE] tex_file = 'figures/figure_included.tikz' [/STATE] [STATE] tex_file = 'figures/figure_included.tex' [/STATE]
      logging.info('Replacing \\includesvg calls in file %s.', tex_file)
      tex_contents[tex_file] = _replace_includesvg(
          tex_contents[tex_file], splits['svg_inkscape']
      )

    for tex_file in tex_contents: #  [STATE] tex_file = 'main.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included.tex' [/STATE] [STATE] tex_file = 'figures/figure_not_included_2.tex' [/STATE] [STATE] tex_file = 'figures/figure_included.tikz' [/STATE] [STATE] tex_file = 'figures/figure_included.tex' [/STATE]
      logging.info('Replacing Tikz Pictures in file %s.', tex_file)
      content = _replace_tikzpictures( #  [STATE] content = '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n' [/STATE] [STATE] content = '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n' [/STATE] [STATE] content = '' [/STATE] [STATE] content = '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n' [/STATE] [STATE] content = '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n' [/STATE]
          tex_contents[tex_file], splits['external_tikz_figures']
      )
      # If file ends with '\n' already, the split in last line would add an extra
      # '\n', so we remove it.
      tex_contents[tex_file] = content.split('\n') #  [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}', '\\addplot{figures/data_included.txt}', '']} [/STATE]

    _keep_only_referenced_tex(tex_contents, splits) # [STATE] splits = {'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': [], 'tex_to_copy': ['figures/figure_included.tex', 'figures/figure_included.tikz', 'main.tex']} [/STATE]
    _add_root_tex_files(splits)

    for tex_file in splits['tex_to_copy']: #  [STATE] tex_file = 'figures/figure_included.tikz' [/STATE] [STATE] tex_file = 'main.tex' [/STATE]
      logging.info('Replacing patterns in file %s.', tex_file)
      content = '\n'.join(tex_contents[tex_file]) #  [STATE] content = '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n' [/STATE] [STATE] content = '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n' [/STATE]
      content = _find_and_replace_patterns(
          content, parameters.get('patterns_and_insertions', list())
      )
      tex_contents[tex_file] = content #  [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE] [STATE] tex_contents = {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'} [/STATE]
      new_path = os.path.join(parameters['output_folder'], tex_file) #  [STATE] new_path = '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv/figures/figure_included.tex' [/STATE] [STATE] new_path = '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv/figures/figure_included.tikz' [/STATE] [STATE] new_path = '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv/main.tex' [/STATE]
      logging.info('Writing modified contents to %s.', new_path)
      _write_file_content(
          content,
          new_path,
      )

    full_content = '\n'.join( # [STATE] full_content = '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n\n\ufeff\\includegraphics{ext_tikz/test2.pdf}\n\n\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n' [/STATE]
        ''.join(tex_contents[fn]) for fn in splits['tex_to_copy']
    )
    _copy_only_referenced_non_tex_not_in_root(parameters, full_content, splits)
    for non_tex_file in splits['non_tex_in_root']: #  [STATE] non_tex_file = 'main.bbl' [/STATE]
      logging.info('Copying non-tex file %s.', non_tex_file)
      _copy_file(non_tex_file, parameters)

    _resize_and_copy_figures_if_referenced(parameters, full_content, splits)
    logging.info('Outputs written to %s', parameters['output_folder'])

run_arxiv_cleaner({'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False})

1: def run_arxiv_cleaner(parameters):
2:   """Core of the code, runs the actual arXiv cleaner."""
3:
4:   files_to_delete = [
5:       r'\.aux$',
6:       r'\.sh$',
7:       r'\.blg$',
8:       r'\.brf$',
9:       r'\.log$',
10:       r'\.out$',
11:       r'\.ps$',
12:       r'\.dvi$',
13:       r'\.synctex.gz$',
14:       '~$',
15:       r'\.backup$',
16:       r'\.gitignore$',
17:       r'\.DS_Store$',
18:       r'\.svg$',
19:       r'^\.idea',
20:       r'\.dpth$',
21:       r'\.md5$',
22:       r'\.dep$',
23:       r'\.auxlock$',
24:       r'\.fls$',
25:       r'\.fdb_latexmk$',
26:   ]
27:
28:   if not parameters['keep_bib']:
29:     files_to_delete.append(r'\.bib$')
30:
31:   parameters.update({
32:       'to_delete': files_to_delete,
33:       'figures_to_copy_if_referenced': [
34:           r'\.png$',
35:           r'\.jpg$',
36:           r'\.jpeg$',
37:           r'\.pdf$',
38:       ],
39:   })
40:
41:   logging.info('Collecting file structure.')
42:   parameters['output_folder'] = _create_out_folder(parameters['input_folder'])
43:
44:   from_zip = parameters['input_folder'].endswith('.zip')
45:   tempdir_context = (
46:       tempfile.TemporaryDirectory() if from_zip else contextlib.suppress()
47:   )
48:
49:   with tempdir_context as tempdir:
50:
51:     if from_zip:
52:       logging.info('Unzipping input folder.')
53:       shutil.unpack_archive(parameters['input_folder'], tempdir)
54:       parameters['input_folder'] = tempdir
55:
56:     splits = _split_all_files(parameters)
57:
58:     logging.info('Reading all tex files')
59:     tex_contents = _read_all_tex_contents(
60:         splits['tex_in_root'] + splits['tex_not_in_root'], parameters
61:     )
62:
63:     for tex_file in tex_contents:
64:       logging.info('Removing comments in file %s.', tex_file)
65:       tex_contents[tex_file] = _remove_comments_and_commands_to_delete(
66:           tex_contents[tex_file], parameters
67:       )
68:
69:     for tex_file in tex_contents:
70:       logging.info('Replacing \\includesvg calls in file %s.', tex_file)
71:       tex_contents[tex_file] = _replace_includesvg(
72:           tex_contents[tex_file], splits['svg_inkscape']
73:       )
74:
75:     for tex_file in tex_contents:
76:       logging.info('Replacing Tikz Pictures in file %s.', tex_file)
77:       content = _replace_tikzpictures(
78:           tex_contents[tex_file], splits['external_tikz_figures']
79:       )
80:       # If file ends with '\n' already, the split in last line would add an extra
81:       # '\n', so we remove it.
82:       tex_contents[tex_file] = content.split('\n')
83:
84:     _keep_only_referenced_tex(tex_contents, splits)
85:     _add_root_tex_files(splits)
86:
87:     for tex_file in splits['tex_to_copy']:
88:       logging.info('Replacing patterns in file %s.', tex_file)
89:       content = '\n'.join(tex_contents[tex_file])
90:       content = _find_and_replace_patterns(
91:           content, parameters.get('patterns_and_insertions', list())
92:       )
93:       tex_contents[tex_file] = content
94:       new_path = os.path.join(parameters['output_folder'], tex_file)
95:       logging.info('Writing modified contents to %s.', new_path)
96:       _write_file_content(
97:           content,
98:           new_path,
99:       )
100:
101:     full_content = '\n'.join(
102:         ''.join(tex_contents[fn]) for fn in splits['tex_to_copy']
103:     )
104:     _copy_only_referenced_non_tex_not_in_root(parameters, full_content, splits)
105:     for non_tex_file in splits['non_tex_in_root']:
106:       logging.info('Copying non-tex file %s.', non_tex_file)
107:       _copy_file(non_tex_file, parameters)
108:
109:     _resize_and_copy_figures_if_referenced(parameters, full_content, splits)
110:     logging.info('Outputs written to %s', parameters['output_folder'])
111:
112: run_arxiv_cleaner({'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False})

Question: What is the value of ' tex_contents ' in line '65' after '2' th iteration when 'run_arxiv_cleaner({'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False})' is executed?

Answer: {'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width


def _read_all_tex_contents(tex_files, parameters):
  contents = {} # [STATE] contents = {} [/STATE]
  for fn in tex_files: #  [STATE] fn = 'main.tex' [/STATE] [STATE] fn = 'figures/figure_not_included.tex' [/STATE] [STATE] fn = 'figures/figure_not_included_2.tex' [/STATE] [STATE] fn = 'figures/figure_included.tikz' [/STATE] [STATE] fn = 'figures/figure_included.tex' [/STATE]
    contents[fn] = _read_file_content( #  [STATE] contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n']} [/STATE] [STATE] contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n']} [/STATE] [STATE] contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': []} [/STATE] [STATE] contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}']} [/STATE] [STATE] contents = {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']} [/STATE]
        os.path.join(parameters['input_folder'], fn)
    )
  return contents

_read_all_tex_contents(['main.tex', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], {'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'})

1: def _read_all_tex_contents(tex_files, parameters):
2:   contents = {}
3:   for fn in tex_files:
4:     contents[fn] = _read_file_content(
5:         os.path.join(parameters['input_folder'], fn)
6:     )
7:   return contents
8:
9: _read_all_tex_contents(['main.tex', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], {'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'})

Question: What is the value of ' contents ' in line '4' after '2' th iteration when '_read_all_tex_contents(['main.tex', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], {'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/XXX/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'})' is executed?

Answer: {'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width


def _strip_tex_contents(lines, end_str):
  """Removes everything after end_str."""
  for i in range(len(lines)): #  [STATE] i = 0 [/STATE] [STATE] i = 1 [/STATE] [STATE] i = 2 [/STATE] [STATE] i = 3 [/STATE] [STATE] i = 4 [/STATE] [STATE] i = 5 [/STATE] [STATE] i = 6 [/STATE] [STATE] i = 7 [/STATE] [STATE] i = 8 [/STATE] [STATE] i = 9 [/STATE] [STATE] i = 10 [/STATE] [STATE] i = 11 [/STATE] [STATE] i = 12 [/STATE] [STATE] i = 13 [/STATE] [STATE] i = 14 [/STATE] [STATE] i = 15 [/STATE] [STATE] i = 16 [/STATE] [STATE] i = 17 [/STATE] [STATE] i = 18 [/STATE] [STATE] i = 19 [/STATE] [STATE] i = 20 [/STATE]
    if end_str in lines[i]:
      if '%' not in lines[i]:
        return lines[: i + 1]
      elif lines[i].index('%') > lines[i].index(end_str):
        return lines[: i + 1]
  return lines

_strip_tex_contents(['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n', '\n', 'This should be ignored.\n'], '\\end{document}')

1: def _strip_tex_contents(lines, end_str):
2:   """Removes everything after end_str."""
3:   for i in range(len(lines)):
4:     if end_str in lines[i]:
5:       if '%' not in lines[i]:
6:         return lines[: i + 1]
7:       elif lines[i].index('%') > lines[i].index(end_str):
8:         return lines[: i + 1]
9:   return lines
10:
11: _strip_tex_contents(['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n', '\n', 'This should be ignored.\n'], '\\end{document}')

Question: What is the value of ' i ' in line '3' after '17' th iteration when '_strip_tex_contents(['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n', '\n', 'This should be ignored.\n'], '\\end{document}')' is executed?

Answer: 16