Commit 9dcd93ce authored by Matthieu Boileau's avatar Matthieu Boileau

Dev doit

parent 90cf5c6a
......@@ -5,6 +5,6 @@
__pycache__/
.idea/
.vscode/
venv/
.env/
build/
notebooks/
\ No newline at end of file
notebooks/
[submodule "skeleton/reveal.js"]
path = skeleton/reveal.js
[submodule "reveal.js"]
path = reveal.js
url = https://github.com/hakimel/reveal.js.git
branch = 3.7.0
......@@ -6,24 +6,37 @@ import argparse
from .nbcourse import NbCourse, DEFAULT_CONFIG_FILE
from .quickstart import initialize
from pathlib import Path
from .mydoit import MyDoit, ClassTaskLoader
import sys
def main():
parser = argparse.ArgumentParser(description=__name__.__doc__)
course = NbCourse()
loader = ClassTaskLoader(course)
prog = globals()['__package__']
description = globals()['__doc__']
formatter = argparse.RawDescriptionHelpFormatter
epilog = MyDoit(loader).get_help()
parser = argparse.ArgumentParser(prog=prog,
description=description,
formatter_class=formatter,
epilog=epilog)
init = parser.add_argument_group('initialize')
init.add_argument('--init', action='store_true',
help=initialize.__doc__)
generate = parser.add_argument_group('generate')
generate.add_argument('--config', type=Path, default=DEFAULT_CONFIG_FILE,
help='load YAML file containing site configuration')
generate.add_argument('--book', type=Path,
help='Generate a pdf book using bookbook')
parser.add_argument('--init', type=str, dest='course_title',
help='initiate a nbcourse directory')
args = parser.parse_args()
if args.course_title:
initialize(args.course_title)
generate.add_argument('--config', dest='config_file', type=Path,
default=DEFAULT_CONFIG_FILE,
help='Lad YAML file describing site configuration '
'(default: %(default)s)')
# Handle undefined arguments
parsed, extra = parser.parse_known_args()
if parsed.init:
initialize()
else:
course = NbCourse(args.config)
if args.book:
course.build_book(args.book)
else:
course.build_pages()
course = NbCourse(parsed.config_file)
course.run(extra)
import jinja2
import logging
import os
from bookbook.latex import combine_and_convert
from pathlib import Path
logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)
class BookContext:
"""
A context manager to work in a directory with a temporary file
"""
def __init__(self, filename, filecontent: str, work_path: Path):
self.filepath = Path(filename)
self.filecontent = filecontent
self.work_path = work_path
self.curdir = None
def __enter__(self):
self.initial_path = Path.cwd()
os.chdir(self.work_path)
with open(self.filepath, 'w') as f:
f.write(self.filecontent)
return self
def __exit__(self, *args):
os.remove(self.filepath)
os.chdir(self.initial_path)
class Book:
"""A book to be built with bookbook"""
def __init__(self, conf):
self.output_path = conf['output_path']
self.project_path = conf['project_path']
self.titlepage_path = conf['pages']['path'] / Path("titlepage.tex")
self.book_title = conf['book_title']
self.bookbook_filename = conf['bookbook_filename']
self.template_path = conf['template_path'].as_posix()
def build(self):
"""Build book"""
# Render basic bookbook template including /pages/titlepage.tex content
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.template_path))
template = env.get_template('book.tplx')
with open(self.titlepage_path) as f:
titlepage_content = f.read()
bookbook_template = template.render(titlepage=titlepage_content)
# Work temporarly in self.output_path directory
with BookContext(Path(self.bookbook_filename), bookbook_template,
self.output_path):
# Call bookbook
combine_and_convert(Path('.'),
Path(self.book_title),
pdf=True,
template_file=self.bookbook_filename)
"""Define some Doit customizations"""
from doit.cmd_help import Help
from doit.doit_cmd import DoitMain
from doit.cmd_base import TaskLoader, ModuleTaskLoader
import inspect
class MyDoitMain(DoitMain):
BIN_NAME = globals()['__package__']
class MyHelp(Help):
"""Extend doit Help class to add a get_usage() function"""
def get_usage(self, cmds):
"""return doit "usage" (basic help) instructions"""
msg = "\n"
for cmd_name in sorted(cmds.keys()):
cmd = cmds[cmd_name]
msg += " {:16s} {}\n".format(cmd_name, cmd.doc_purpose)
msg += "\n"
for line in [
" {} show doit's help\n",
" {} task show help on task dictionary fields\n",
" {} <command> show command usage\n",
" {} <task-name> show task usage\n"]:
msg += line.format("help")
return msg
class MyDoit(DoitMain):
"""Extend DoitMain class to add a get_help() function"""
def get_help(self):
"""return help usage as string"""
# get list of available commands
sub_cmds = self.get_cmds()
help = MyHelp(config=self.config,
bin_name='',
cmds=sub_cmds)
return help.get_usage(sub_cmds.to_dict())
class ClassTaskLoader(ModuleTaskLoader):
"""Implementation of a loader of tasks from a Class namespace"""
def load_tasks(self, cmd, params, args):
# Build dict from class namespace
self.mod_dict = dict(inspect.getmembers(self.mod_dict))
return self._load_from(cmd, self.mod_dict, self.cmd_names)
This diff is collapsed.
import nbformat
import re
import logging
import markdown
from pathlib import Path
from pprint import pformat
import jinja2
from bs4 import BeautifulSoup
log = logging.getLogger(__name__)
class HomePage:
"""A class to handle a homepage to be rendered"""
html_template = "index.html"
def __init__(self, nbcourse):
conf = nbcourse.conf
self.html = conf['output_path'] / conf['pages']['home']['name']
self.title = conf['title']
self.notebooks = nbcourse.notebooks
self.parent = None
self.variables = conf
self.template_path = conf['template_path']
self.chapters = {}
def _get_chapter(self, path: Path):
"""Get chapter from notebook file (source: bookbook)"""
chapter_no = int(re.match(r'(\d+)\-', path.stem).group(1))
nb = nbformat.read(str(path), as_version=4)
assert nb.cells[0].cell_type == 'markdown', nb.cells[0].cell_type
lines = nb.cells[0].source.splitlines()
if lines[0].startswith('# '):
header = lines[0][2:]
elif len(lines) > 1 and lines[1].startswith('==='):
header = lines[0]
else:
assert False, "No heading found in {}".format(path)
assert path.suffix == '.ipynb', path
return {'number': chapter_no,
'title': header,
'filename': path.stem}
def _get_variables(self):
"""Set some variables from conf dictionnary and notebooks files"""
logging.basicConfig(level=logging.INFO)
# Get chapters from notebook files
chapters = []
for nbfile in self.notebooks:
chapter = self._get_chapter(nbfile)
chapter['preview_only'] = chapter['number'] in \
self.variables['chapter_preview_only']
chapters.append(chapter)
chapters.sort(key=lambda chapter: chapter['number'])
self.variables.update({'chapters': chapters})
log.info("Homepage template variables:")
log.info(pformat(self.variables))
def _render_template(self):
"""Return html rendered from template and variables"""
# Inject variables into html_template
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.template_path.as_posix()))
template = env.get_template(self.html_template)
html_out = template.render(self.variables)
with open(self.html, 'w') as f:
f.write(html_out)
def render(self):
"""Render html using templating"""
self._get_variables()
self._render_template()
class MarkdownPage(HomePage):
html_template = "article.html"
def __init__(self, nbcourse, html, title, src, parent=None):
super().__init__(nbcourse)
self.html = nbcourse.conf['output_path'] / html
self.title = title
self.src = nbcourse.conf['pages']['path'] / Path(src)
self.parent = parent
def render(self):
"""Render markdown file into html file adding ToC"""
pattern = re.compile(r'^doc\/(.*)\.md$')
def markdown2html():
"""Return html from markdown file"""
with open(self.src, 'r') as f:
text = f.read()
md = markdown.Markdown(extensions=['fenced_code', 'codehilite',
'toc'])
body = md.convert(text)
# Transform link to file.md into link to file.html
soup = BeautifulSoup(body, 'html.parser')
for link in soup.findAll(['a']):
href = link.get('href')
link['href'] = pattern.sub(r'\1.html', href)
return soup.prettify(), md.toc
def get_menu_list():
"""Return a list to be displayed as a top menu"""
# At least current page
menu = [(self.title, self.html)]
page = self
while page.parent.parent:
menu.append((page.parent.title, page.parent.html))
page = page.parent
menu.reverse()
return menu
body, toc = markdown2html()
self.variables = {'title': self.title,
'article_content': body,
'toc': toc,
'menu': get_menu_list()}
self._render_template()
import os
from distutils.dir_util import copy_tree
from pathlib import Path
SKEL_PATH = Path(os.path.dirname(os.path.abspath(__file__))) / \
Path("../skeleton")
SKEL_PATH = Path(__file__).absolute().parents[1] / Path("skeleton")
def initialize(course_title):
def initialize():
"""Initialize a nbcourse directory"""
copy_tree(SKEL_PATH, os.getcwd())
makefile_inc = """\
course_title = {}
config_file = nbcourse.yml
notebook_dir = notebooks
output_dir = build
theme_dir = theme/default
""".format(course_title)
with open("Makefile.inc", 'w') as f:
f.write(makefile_inc)
../reveal.js
\ No newline at end of file
import shutil
from pathlib import Path
from zipfile import ZipFile
import os
IGNORED = '__pycache__'
def update_material(src: Path, dst: Path):
"""Remove dst tree then update with src"""
shutil.rmtree(dst, ignore_errors=True)
shutil.copytree(src, dst, ignore=shutil.ignore_patterns(IGNORED))
def get_file_list(path: Path, relative: bool = True, ignore: list = []):
"""Return a list of file paths relative to path"""
ignored = set([IGNORED]) | set(ignore)
def get_path(obj):
"""return a relative or absolute path"""
return obj.relative_to(path) if relative else obj
return [get_path(obj) for obj in path.glob('**/*')
if (obj.is_file() and ignored.isdisjoint(obj.parts))]
def clean_tree(target):
"""Clean tree ignoring errors"""
shutil.rmtree(target, ignore_errors=True)
def update_dict(current: dict, new: dict):
"""Update current multi-level dict with new dict"""
for k, v in new.items():
if k in current and type(v) is dict:
# Update current entry by descending in sub dict
update_dict(current[k], v)
else:
# Add new entry or update with new value
current[k] = v
def zip_files(zip_file_name: Path, paths_to_zip: list, refpath: Path = None,
arcdir: str = None):
"""Archive files_to_zip in zip file"""
with ZipFile(zip_file_name, 'w') as zf:
for path in paths_to_zip:
# Convert refpath/file in arcdir/file if asked
arcpath = arcdir / path.relative_to(refpath) if refpath else None
zf.write(path, arcname=arcpath)
def clean_file(file):
"""Remove file ignoring errors"""
try:
os.remove(file)
except OSError:
pass
Subproject commit 2c5396b7d347f8ee1344016f15b93d4f78401569
include Makefile.inc
notebooks := $(wildcard $(notebook_dir)/*.ipynb)
makefile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
project_dir := $(dir $(makefile_path))
template_dir := $(project_dir)/$(theme_dir)/templates/
local_reveal := False
ifeq ($(local_reveal),True)
# Useful for running in local with internet connexion
revealprefix := "reveal.js"
else
# Needed for online publication by gitlab pages
revealprefix := "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0"
endif
executed_notebooks := $(addprefix $(output_dir)/, $(notdir $(notebooks)))
html := $(addprefix $(output_dir)/, $(notdir $(subst .ipynb,.html,$(notebooks))))
slides := $(addprefix $(output_dir)/, $(notdir $(subst .ipynb,.slides.html,$(notebooks))))
archives := $(addprefix $(output_dir)/, $(subst .ipynb,.zip,$(notdir $(notebooks))))
pages_md := $(wildcard pages/*.md)
pages := $(output_dir)/index.html\
$(addprefix $(output_dir)/, $(notdir $(subst .md,.html,$(pages_md))))
.PHONY: all clean html slides executed_notebooks pages copy_to_build pdf archives archive install
all: $(output_dir) html slides pages archives pdf
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " slides to make slideshows (use local_reveal=True \
to run them without internet connection)"
@echo " pdf to compile all notebooks as a single PDF book"
@echo " archives to make ##-notebook.zip files"
@echo " archive to make $(course_title).zip file"
@echo " pages to make index and other pages"
@echo "Use \`make' to run all these targets"
install:
pip install -U --user -r requirements.txt
executed_notebooks: copy_to_build $(executed_notebooks)
html: copy_to_build $(html)
slides: copy_reveal $(slides)
pages: copy_to_build $(pages)
pdf: copy_to_build $(output_dir)/$(course_title).pdf
archives: $(output_dir) $(archives)
archive: $(output_dir)/$(course_title).zip
define nbconvert
jupyter nbconvert --to $(1) $< --output-dir=$(output_dir)
endef
$(output_dir):
@mkdir -p $(output_dir)
copy_to_build: $(output_dir)
rsync -ra --delete $(notebook_dir)/fig $(output_dir)/ --exclude ".*/" --exclude "__pycache__"
rsync -ra --delete $(notebook_dir)/exos $(output_dir)/ --exclude ".*/" --exclude "__pycache__"
rsync -ra --delete $(theme_dir)/css $(output_dir)/
rsync -ra --delete $(theme_dir)/img $(output_dir)/
copy_reveal: $(output_dir)
rsync -ra --delete reveal.js $(output_dir)/
$(executed_notebooks): $(output_dir)/%.ipynb: $(notebook_dir)/%.ipynb
$(call nbconvert,notebook,$<) --execute --allow-errors --ExecutePreprocessor.timeout=60
$(pages): $(pages_md) $(wildcard $(theme_dir)/img/*) $(wildcard $(theme_dir)/css/*) $(wildcard $(theme_dir)/templates/*) $(CONFIG)
cd $(output_dir) && python3 -m nbcourse --config ../$(config_file)
$(output_dir)/%.html: $(output_dir)/%.ipynb
$(call nbconvert,html,$<)
$(output_dir)/%.slides.html: $(output_dir)/%.ipynb
$(call nbconvert,slides,$<) --reveal-prefix $(revealprefix)
$(output_dir)/$(course_title).pdf: $(executed_notebooks) $(template_dir)/book.tplx
cd $(output_dir) && python3 -m nbcourse --book $(course_title).pdf --config ../nbcourse.yml
$(output_dir)/%.zip: $(notebook_dir)/%.ipynb
zip -r $@ $< $(notebook_dir)/fig $(notebook_dir)/exos --exclude "*/\.*" "*__pycache__*"
$(output_dir)/$(course_title).zip: all
cd $(output_dir) && zip -r $(course_title).zip * --exclude "*/\.*" "*__pycache__*" "*.e" "*.zip"
clean:
rm -rf $(output_dir)
import os
import shutil
from pathlib import Path
from doit.tools import result_dep
from doit.task import clean_targets
PROJECT_DIR = '.'
OUTPUT_DIR = Path('build')
NOTEBOOK_DIR = Path('notebooks')
NOTEBOOK_MATERIAL = 'fig', 'exos'
NOTEBOOK_MATERIAL_PATHS = [NOTEBOOK_DIR / Path(dir)
for dir in NOTEBOOK_MATERIAL]
THEME_DIR = Path("theme/default")
THEME_MATERIAL_DIRNAMES = 'css', 'img'
NBDIR = Path('notebooks')
NOTEBOOKS = tuple(NBDIR.glob('*-*.ipynb'))
REVEAL_PREFIX = 'reveal.js'
PAGES_DIR = Path('pages')
CONFIG_FILE = 'nbcourse.yml'
COURSE_TITLE = 'cours-python'
ZIP_OPTS = r'--exclude "*/\.*" "*__pycache__*"'
TASKS = ('build_book',)
BOOK_TITLE = Path(COURSE_TITLE).with_suffix('.pdf')
IGNORED = '__pycache__'
def make_output_dir():
os.makedirs(OUTPUT_DIR, exist_ok=True)
def task_output_dir():
"""Create empty output directory"""
return {
'targets': [OUTPUT_DIR],
'actions': [make_output_dir],
'uptodate': [OUTPUT_DIR.is_dir],
'clean': True,
}
def update_material(src, dst):
"""Remove dst tree then update with src"""
shutil.rmtree(dst, ignore_errors=True)
shutil.copytree(src, dst, ignore=shutil.ignore_patterns(IGNORED))
def get_file_list(parent_dir, dir):
"""Return a tuple for source and destination"""
src = parent_dir / Path(dir)
src_file_list = [obj for obj in src.glob('**/*')
if (obj.is_file() and IGNORED not in obj.parts)]
dst = OUTPUT_DIR / Path(dir)
dst_file_list = [OUTPUT_DIR / file.relative_to(parent_dir)
for file in src_file_list]
return src, src_file_list, dst, dst_file_list
def clean_tree(target):
shutil.rmtree(target, ignore_errors=True)
def task_copy_notebook_material():
"""Copy notebook material to output directory"""
for dir in NOTEBOOK_MATERIAL:
src, src_file_list, dst, dst_file_list = get_file_list(NOTEBOOK_DIR,
dir)
yield {
'name': dst,
'file_dep': src_file_list,
'targets': dst_file_list,
'actions': [(update_material, (src, dst))],
'clean': [(clean_tree, (dst,))],
}
def task_copy_theme_material():
"""Copy theme material to output directory"""
for dir in THEME_MATERIAL_DIRNAMES:
src, src_file_list, dst, dst_file_list = get_file_list(THEME_DIR,
dir)
yield {
'name': dst,
'file_dep': src_file_list,
'targets': dst_file_list,
'actions': [(update_material, (src, dst))],
'clean': [(clean_tree, (dst,))],
}
def task_copy_reveal():
"""Copy reveal.js directory to output directory"""
src, src_file_list, dst, dst_file_list = get_file_list(PROJECT_DIR,
"reveal.js")
return {
'file_dep': src_file_list,
'targets': dst_file_list,
'actions': [(update_material, (src, dst))],
'clean': [(clean_tree, (dst,))],
}
def get_nbconvert_cmd(notebook, to_opt, opts=''):
"""Return nbconvert command as string"""
cmd = f"jupyter nbconvert --to {to_opt} {notebook} " + \
f"--output-dir={OUTPUT_DIR} {opts}"
return cmd
def task_execute_notebooks():
"""Write executed notebooks to output directory"""
opts = '--execute --allow-errors --ExecutePreprocessor.timeout=60'
for notebook in NOTEBOOKS:
executed_notebook = OUTPUT_DIR / notebook.relative_to(NBDIR)
cmd = get_nbconvert_cmd(notebook, 'notebook', opts)
yield {
'name': executed_notebook,
'file_dep': [notebook],
'targets': [executed_notebook],
'clean': True,
'actions': [cmd],
}
def task_convert_to_html():
"""Convert executed notebook to html page"""
for notebook in NOTEBOOKS:
executed_notebook = OUTPUT_DIR / notebook.relative_to(NBDIR)
html = OUTPUT_DIR / notebook.relative_to(NBDIR).with_suffix('.html')
cmd = get_nbconvert_cmd(notebook, 'html')
yield {
'name': html,
'file_dep': [executed_notebook],
'targets': [html],
'clean': True,
'actions': [cmd],
}
def task_convert_to_slides():
"""Convert executed notebook to html slides"""