Commit a0147a2a authored by gouarin's avatar gouarin
Browse files

add rst generator

parent 18c1b8d1
from ruamel.yaml import YAML
from ruamel.yaml.reader import Reader
import re
to_remove = [
def strip_invalid(s):
res = ''
for x in s:
if Reader.NON_PRINTABLE.match(x):
# res += '\\x{:x}'.format(ord(x))
res += x
return res
def clean_titles(s):
Add simple quotes to titles to avoid interpretation of ":" as yaml syntax
def title_replace(matchobj):
"""Return a clean title line"""
title_type =
title =
if title.startswith("|-"):
# Do not replace if multiline content
# Avoid real text simple quote to be interpreted as end of string
title = re.sub(r"'", "''", title)
return f"{title_type}'{title}'"
s = re.sub('^( titre: )(.*)$', title_replace, s, flags=re.MULTILINE)
s = re.sub('^( nom_site: )(.*)$', title_replace, s, flags=re.MULTILINE)
s = re.sub('^( texte: )(.*)$', title_replace, s, flags=re.MULTILINE)
return s
yaml = YAML()
import codecs
last_position = -1
def mixed_decoder(unicode_error):
global last_position
string = unicode_error.object
position = unicode_error.start
new = string.decode("iso-8859-1")
return new[position:unicode_error.end], unicode_error.end
def mixed_decoder_utf8(unicode_error):
global last_position
string = unicode_error.object
position = unicode_error.start
new = string[position:unicode_error.end].encode("utf-8")
return new, unicode_error.end
codecs.register_error("mixed", mixed_decoder)
codecs.register_error("utf8", mixed_decoder_utf8)
with open("./spip_yml/spip_articles.yml", 'r') as stream:
lines =
def remove(match):
return ''
for r in to_remove:
regex = re.compile(f'({r}.*)')
lines = regex.sub(remove, lines)
lines = lines.encode('iso-8859-1', 'utf8').decode('utf-8', 'mixed')
y = yaml.load(clean_titles(strip_invalid(lines)))
with open("./spip_yml/spip_articles_clean.yml", 'w') as stream:
yaml.dump(y, stream)
......@@ -9,6 +9,7 @@ import anytree
import argparse
from ruamel.yaml import YAML
import bs4
from bs4 import BeautifulSoup
from colorlog import ColoredFormatter
import ftfy
import logging
......@@ -299,17 +300,329 @@ class SpipToMarkdown:
class SpipToRst(SpipToMarkdown):
"""A class to export spip article format to a ReStructuredText Pelican article"""
def convert(self, s, preserve_line_breaks=False):
"""Apply a pandoc conversion to markdown format"""
s = super().convert(s)
if preserve_line_breaks:
extra_args = ['--wrap=preserve']
extra_args = ['--wrap=auto']
s = pypandoc.convert_text(s, 'rst', format='md', extra_args=extra_args)
s = re.sub(r"%7Bfilename%7D", r"{filename}", s) # Correct unwanted pandoc translation
def __init__(self, website): = website
def convert(self, s):
"""Convert string from Spip format to Pelican markdown format"""
# # s = self.html_link(s)
# # s = self.html_img(s)
s = self.ordered_list(s)
s = self.unordered_list(s)
# s = self.horizontal_rule(s)
s = self.fix_li(s)
s = self.convert_html(s)
s = self.remove_font(s)
s = self.bold(s)
s = self.italic(s)
s =
s = self.remove_space(s)
s = self.remove_empty_link(s)
s = self.document(s)
s = self.fix_table(s)
s = self.remove_blank(s)
s = self.header(s)
s = self.header_extended(s)
return s
def document(self, s):
SPIP: <doc|path> or <img|path>
md: [text](url) or ![](img)
def doc_rst(match):
doc_type = match[1]
doc_id = int(match[2])
url = os.path.join(, "IMG",[doc_id])
docname = os.path.basename(url)
if doc_type == 'doc':
return f'`{docname} <{url}>`__'
return f'\n\n..image:: {url}\n\n'
regex = re.compile(r'<(doc|img)([0-9]+)\|.*>')
return regex.sub(doc_rst, s)
def html_link(self, s):
"""Replace html href by the right Pelican (relative) URL"""
def link_replace(matchobj):
"""A call back function to replace a Spip absolute link by a relative link to Pelican file"""
spip_type =
id_art = int(
anchorobj = re.match(r"#(.*)",
if anchorobj:
new_url =
new_url = f"spip_{spip_type}-{id_art}.html"
return new_url
def link_replace_doc(matchobj):
"""Prepend attachment document path with attachment prefix"""
return +
soup = bs4.BeautifulSoup(s, "html.parser")
for link in soup.find_all('a'):
link_url = link.get('href')
if link_url:
new_url = re.sub(r"\A{}/spip.php\?(article|rubrique)([0-9]+)(.*)".format(,
new_url = re.sub(r"\A({}/|)(Documents/.*)".format(, link_replace_doc, new_url)
link['href'] = new_url
return soup.prettify(formatter=None) # formatter=None to avoid ">" -> "&gt;" conversion
def html_img(self, s):
"""Replace html img src by the right Pelican (relative) URL"""
def src_replace(matchobj):
"""Prepend attachment image path with attachment prefix"""
return +
soup = bs4.BeautifulSoup(s, "html.parser")
for img in soup.find_all('img'):
img_src = img.get('src')
if img_src:
new_src = re.sub(r"\ADocuments/.*", src_replace, img_src)
img['src'] = new_src
return soup.prettify(formatter=None) # formatter=None to avoid ">" -> "&gt;" conversion
def fix_table(self, s):
def remove_bad_char(match):
return '| |'
regex = re.compile('\|(\^|<)\|')
s = regex.sub(remove_bad_char, s)
return re.sub(r'\|', '', s)
def fix_li(self, s):
soup = BeautifulSoup(s, 'html.parser')
for li in soup.find_all('li'):
if isinstance(li.contents[0], str):
text = li.contents[0].replace('\n', '')
return soup.prettify(formatter=None)
def remove_space(self, s):
new = []
for l in s.split("\n"):
return '\n'.join(new)
def remove_blank(self, s):
new = []
for l in s.split("\n"):
if l.lstrip().startswith('-'):
return '\n'.join(new)
def convert_html(self, lines):
soup = BeautifulSoup(lines, 'html.parser')
for html in soup.find_all('ul'):
s = pypandoc.convert_text(html, 'rst', format='html', extra_args=['--wrap=preserve'])
for html in soup.find_all('a'):
s = pypandoc.convert_text(html, 'rst', format='html', extra_args=['--wrap=preserve'])
return soup.prettify(formatter=None)
def remove_font(s):
def font_rst(match):
return ' '
regex = re.compile(r'(<font .*>)')
s = regex.sub(font_rst, s)
regex = re.compile(r'(</font>)')
s = regex.sub(font_rst, s)
regex = re.compile(r'(<html>)')
s = regex.sub(font_rst, s)
regex = re.compile(r'(</html>)')
s = regex.sub(font_rst, s)
regex = re.compile(r'(<hr/>)')
return regex.sub(font_rst, s)
def bold(s):
SPIP: {{ ... }}
md: **...**
def bold_rst(match):
text = match[2].strip()
return f'**{text}** '
regex = re.compile(r'({{2})([^}]+)(}{2})')
new = []
for l in s.split("\n"):
new.append(regex.sub(bold_rst, l))
return '\n'.join(new)
def italic(s):
SPIP: {...}
md: *...*
def italic_rst(match):
text = match[2].strip()
return f'*{text}* '
regex = re.compile(r'({)([^}]*)(})')
new = []
for l in s.split("\n"):
new.append(regex.sub(italic_rst, l))
return '\n'.join(new)
def ordered_list(s):
SPIP: - or -# in 1rst level, -## for second level, etc.
md: 1. with 4-space indents
def ordered_rst(match):
indent = ' '*4*(match[1].count('*')-1)
return f'\n{indent}- {match[2]}\n'
regex = re.compile(r'^\s*-\s*(\#*)(.*)')
new = []
for l in s.split("\n"):
new.append(regex.sub(ordered_rst, l))
return '\n'.join(new)
def remove_empty_link(s):
def replace(match):
return f'{match[1]}'
regex = re.compile(r'`(.*)<>`__')
return regex.sub(replace, s)
def unordered_list(s):
SPIP: - or -* in 1rst level, -** for second level, etc.
md: - with 4-space indents
def unordered_rst(match):
indent = ' '*4*(match[1].count('*')-1)
text = match[2].strip()
return f'\n{indent}- {text}\n'
regex = re.compile('^\s*-\s*(\**)(.*)')
new = []
for l in s.split("\n"):
new.append(regex.sub(unordered_rst, l))
return '\n'.join(new)
def header(s):
SPIP: {{{...}}}
md: ## ...
def header_rst(match):
text = match[2].strip()
return text + '\n' + '='*len(text) + '\n'
regex = re.compile(r'({{3})([^}]*)(}{3})')
return regex.sub(header_rst, s)
def header_extended(s):
SPIP: {{{{{...}}}}}
md: ### ...
def header_rst(match):
text = match[2].strip()
return text + '\n' + '-'*len(text) + '\n'
regex = re.compile(r'({{5})([^}]*)(}{5})')
return regex.sub(header_rst, s)
def horizontal_rule(s):
SPIP: ---- with no carriage return before and after
md: \n---\n
s = re.sub(r"<hr>", r"\n---\n", s)
s = re.sub(r"----", r"\n---\n", s)
return s
def link(self, s):
SPIP: [text->url] or [text -> url]
md: [text](url) or <url> if text is empty
def nullify_url(id_art, text, url):
"""Throw WARNING message and return empty URL"""
msg = f" WARNING: nullify link to non existing article {id_art}\n"
msg += f" text: {text}\n"
msg += f" url: {url}"
logger.warning(msg) += 1
return ""
def link_rst(match):
text = match[1]
link = match[2].strip()
if text == '' or text == link:
return f'{link}'
email = re.match('mailto:(.*)', link)
if email:
return f'{}'
http_url = re.match(r'http', link)
if http_url:
return f'`{text} <{link}>`__'
doc_url = re.match(r'doc([0-9]+)', link)
if doc_url:
link = os.path.join(, "IMG",[int(])
return f'`{text} <{link}>`__'
art_url = re.match(r"(art|rub|brev)([0-9]+)", link)
if art_url:
art_id =
link =[art_id]
except KeyError:
link = nullify_url(art_id, text, link)
return f'`{text} <{link}>`__'
link = os.path.join(, link)
return f'`{text} <{link}>`__'
regex = re.compile(r'\[([^]]*)\s*-\s*>\s*([^]]*)\]')
return regex.sub(link_rst, s)
class Article:
"""A generic class for a single Spip article or rubrique to be converted into a Pelican article file"""
......@@ -319,6 +632,7 @@ class Article:
self.type = spip_type = website
id_tag = 'id_' + self.type
self.short_id = spip_article[id_tag] = f"{SHORTEN[self.type]}{self.short_id}"
self.title = spip_article['titre']
......@@ -451,13 +765,14 @@ class ArticleRst(Article):
def convert_title(self, title):
"""Prevent line breaks when converting title"""
return self.convert(title, preserve_line_breaks=True).strip()
return self.convert(title).strip()
def get_header(self):
"""Return header in rst format"""
title = f"{self.title}\n{'#'*len(self.title)}\n\n"
header = title + f"""\
#title = f"{self.title}\n{'#'*len(self.title)}\n\n"
header = f"""\
:title: {self.title}
:date: {}
:modified: {self.modified}
:category: {self.category}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment