add rst generator

create_spip_article_clean.py

+from ruamel.yaml import YAML
+from ruamel.yaml.reader import Reader
+import re
+
+to_remove = [
+    'surtitre:',
+    'soustitre:',
+    'chapo:',
+    'maj:',
+    'export:',
+    'visites:',
+    'referers:',
+    'popularite:',
+    'accepter_forum:',
+    'date_modif:',
+    'langue_choisie:',
+    'id_trad:',
+    'id_version:',
+    'nom_site:',
+    'url_site:', 
+    'virtuel:',
+    'date_redac:'
+]
+
+def strip_invalid(s):
+    res = ''
+    for x in s:
+        if Reader.NON_PRINTABLE.match(x):
+            # res += '\\x{:x}'.format(ord(x))
+            continue
+        res += x
+    return res
+
+def clean_titles(s):
+    """
+    Add simple quotes to titles to avoid interpretation of ":" as yaml syntax
+    """
+    def title_replace(matchobj):
+        """Return a clean title line"""
+        title_type = matchobj.group(1)
+        title = matchobj.group(2)
+        if title.startswith("|-"):
+            # Do not replace if multiline content
+            return matchobj.group(0)
+        else:
+            # Avoid real text simple quote to be interpreted as end of string
+            title = re.sub(r"'", "''", title)
+            return f"{title_type}'{title}'"
+
+    s = re.sub('^(  titre: )(.*)$', title_replace, s, flags=re.MULTILINE)
+    s = re.sub('^(  nom_site: )(.*)$', title_replace, s, flags=re.MULTILINE)
+    s = re.sub('^(  texte: )(.*)$', title_replace, s, flags=re.MULTILINE)
+    return s
+
+yaml = YAML()
+
+import codecs
+
+last_position = -1
+
+def mixed_decoder(unicode_error):
+    global last_position
+    string = unicode_error.object
+    position = unicode_error.start
+    new = string.decode("iso-8859-1")
+    return new[position:unicode_error.end], unicode_error.end
+
+def mixed_decoder_utf8(unicode_error):
+    global last_position
+    string = unicode_error.object
+    position = unicode_error.start
+    new = string[position:unicode_error.end].encode("utf-8")
+    return new, unicode_error.end
+
+codecs.register_error("mixed", mixed_decoder)
+codecs.register_error("utf8", mixed_decoder_utf8)
+
+
+with open("./spip_yml/spip_articles.yml", 'r') as stream:
+    lines = stream.read()
+
+    def remove(match):
+        return ''
+
+    for r in to_remove:
+        regex = re.compile(f'({r}.*)')
+        lines = regex.sub(remove, lines)
+
+    lines = lines.encode('iso-8859-1', 'utf8').decode('utf-8', 'mixed')
+
+    y = yaml.load(clean_titles(strip_invalid(lines)))
+
+with open("./spip_yml/spip_articles_clean.yml", 'w') as stream:
+    yaml.dump(y, stream)