Commit c5b2537a authored by Matthieu Boileau's avatar Matthieu Boileau

Unitary tests

parent 752522cd
......@@ -8,3 +8,4 @@ __pycache__/
.env/
build/
notebooks/
tests/notebooks
pytest
\ No newline at end of file
......@@ -4,20 +4,26 @@ Build a small website to host Jupyter notebooks as course chapters
import argparse
from .nbcourse import NbCourse, DEFAULT_CONFIG_FILE
from .quickstart import initialize
from .initialize import initialize
from pathlib import Path
from .mydoit import MyDoit, ClassTaskLoader
def main():
def get_help_epilog() -> str:
"""Instanciate a minimal nbcourse to return help string"""
course = NbCourse()
loader = ClassTaskLoader(course)
return MyDoit(loader).get_help()
def main():
"""CLI for nbcourse"""
prog = globals()['__package__']
description = globals()['__doc__']
formatter = argparse.RawDescriptionHelpFormatter
epilog = MyDoit(loader).get_help()
epilog = get_help_epilog()
parser = argparse.ArgumentParser(prog=prog,
description=description,
formatter_class=formatter,
......@@ -28,7 +34,7 @@ def main():
generate = parser.add_argument_group('generate')
generate.add_argument('--config', dest='config_file', type=Path,
default=DEFAULT_CONFIG_FILE,
help='Lad YAML file describing site configuration '
help='Load YAML file describing site configuration '
'(default: %(default)s)')
# Handle undefined arguments
......@@ -38,4 +44,4 @@ def main():
initialize()
else:
course = NbCourse(parsed.config_file)
course.run(extra)
course.build(extra)
......@@ -27,7 +27,7 @@ class BookContext:
return self
def __exit__(self, *args):
os.remove(self.filepath)
self.filepath.unlink()
os.chdir(self.initial_path)
......@@ -38,7 +38,8 @@ class Book:
self.output_path = conf['output_path']
self.project_path = conf['project_path']
self.titlepage_path = conf['pages']['path'] / conf['pages']['titlepage']
self.titlepage_path = conf['pages']['path'] / \
conf['pages']['titlepage']
self.book_title = conf['book_title']
self.bookbook_filename = conf['bookbook_filename']
self.template_path = conf['template_path'].as_posix()
......
import os
from distutils.dir_util import copy_tree
from pathlib import Path
......@@ -9,4 +8,4 @@ SKEL_PATH = Path(__file__).absolute().parents[1] / Path("skeleton")
def initialize():
"""Initialize a nbcourse directory"""
copy_tree(SKEL_PATH, os.getcwd())
copy_tree(SKEL_PATH, str(Path.cwd()))
......@@ -17,7 +17,10 @@ import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert import HTMLExporter, SlidesExporter
from .mydoit import MyDoitMain, ClassTaskLoader
import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
DEFAULT_CONFIG_FILE = "nbcourse.yml"
......@@ -36,12 +39,21 @@ class NbCourse:
'pages': {
'dir': 'pages',
'titlepage': 'titlepage.tex',
'home': {'name': 'index.html'}
'home': 'index.html'
},
'bookbook_filename': 'bookbook.tex',
'local_reveal': False,
'slug_title': 'course',
'output_dir': 'build',
'title': 'A course',
'links': [{
'title': 'Manual',
'target': 'manual.html',
'icon': {
'path': 'img/Infobox_info_icon.svg',
'width': '35px',
}
}]
}
def __init__(self, config_file: Path = None):
......@@ -49,6 +61,9 @@ class NbCourse:
if config_file:
update_dict(self.conf, self._get_user_config(config_file))
self.conf['project_path'] = Path(config_file).absolute().parent
else:
# Only default config is loaded (only useful for tests)
self.conf['project_path'] = Path.cwd()
self.conf['template_path'] = Path(self.conf['theme']['dir'],
'templates')
for key in 'nb', 'theme':
......@@ -66,7 +81,7 @@ class NbCourse:
'.pdf')
self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
self.book = self.conf['output_path'] / self.conf['book_title']
self.md_pages = list(self.conf['pages']['path'].glob('*.md'))
self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
self.html_pages = self._get_pages() # homepage and documentation pages
self.theme_files = [file for file in
self.conf['theme']['path'].glob('**/*')
......@@ -96,12 +111,10 @@ class NbCourse:
def _get_pages(self):
"""get a list of target html pages from markdown source"""
# Start html_pages list with index.html
html_pages = [self.conf['output_path'] /
self.conf['pages']['home']['name']]
for md_page in self.md_pages:
html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
for path in self.md_page_paths:
html_pages.append(self.conf['output_path'] /
md_page.relative_to(
self.conf['pages']['path']).with_suffix('.html'))
Path(path.name).with_suffix('.html'))
return html_pages
@staticmethod
......@@ -117,28 +130,27 @@ class NbCourse:
Build a mini website using html templating and markdown conversion
"""
def render_children(parent, parent_page):
# List mardkown files
md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
# Instanciate all markdown pages to be rendered
md_pages = [MarkdownPage(self, md_page_path)
for md_page_path in md_page_paths]
def render_children(parent):
"""Recursive function to render children pages from parent page"""
try:
children = parent['children']
for child in children.values():
# Render markdown pages
child_page = MarkdownPage(self, child['html'],
title=child['title'],
src=child['md'],
parent=parent_page)
child_page.render()
render_children(child, child_page)
except KeyError:
# Parent page has no children
return
children = (md_page for md_page in md_pages
if md_page.parent_name == parent.name)
for child in children:
# Render markdown pages
child.render(parent)
render_children(child)
# First build homepage
homepage = HomePage(self)
homepage.render()
# Then build children pages
render_children(self.conf['pages']['home'], homepage)
render_children(homepage)
def build_book(self):
"""Build a pdf book using bookbook"""
......@@ -189,6 +201,11 @@ class NbCourse:
'actions': [(update_material, (src_path, dst_path))],
'clean': [(clean_tree, (dst_path,))],
}
else:
return {
'uptodate': [True],
'actions': []
}
def execute_notebook(self, notebook: Path, executed_notebook: Path):
"""Execute notebook and write result to executed_notebook file"""
......@@ -214,6 +231,7 @@ class NbCourse:
yield {
'name': executed_notebook,
'file_dep': [notebook],
'task_dep': ['copy_material'],
'targets': [executed_notebook],
'clean': True,
'actions': [(self.execute_notebook, (notebook,
......@@ -268,7 +286,7 @@ class NbCourse:
}
def task_zip_chapters(self):
"""Build one zip archive per chapter"""
"""Build zip archives for single chapter downloads"""
for notebook in self.notebooks:
zip_file = self.get_executed_notebook(notebook).with_suffix('.zip')
......@@ -279,6 +297,7 @@ class NbCourse:
yield {
'name': zip_file,
'file_dep': paths_to_zip,
'task_dep': ['copy_material'],
'targets': [zip_file],
'clean': True,
'actions': [(zip_files, (zip_file, paths_to_zip))],
......@@ -287,7 +306,8 @@ class NbCourse:
def task_build_pages(self):
"""Build html pages"""
return {
'file_dep': self.md_pages + self.theme_files,
'file_dep': self.md_page_paths + self.theme_files,
'task_dep': ['copy_material'],
'targets': self.html_pages,
'clean': True,
'actions': [self.build_pages],
......@@ -311,11 +331,14 @@ class NbCourse:
'targets': [self.zip_file],
'clean': True,
'actions': [(zip_files, (self.zip_file, paths_to_zip),
{'refpath': self.conf['output_path'],
'arcdir': self.conf['slug_title']})],
{'ref_to_arc': (self.conf['output_path'],
self.conf['slug_title'])})],
}
def run(self, args: list):
def build(self, args: list = None):
"""Build course using Doit"""
if args is None:
args = []
doit_config = {} # TODO: implement verbosity option
sys.exit(MyDoitMain(ClassTaskLoader(self),
extra_config=doit_config).run(args))
......@@ -6,23 +6,53 @@ from pathlib import Path
from pprint import pformat
import jinja2
from bs4 import BeautifulSoup
import frontmatter
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
class HomePage:
"""A class to handle a homepage to be rendered"""
class Page:
"""An abstract class for a static web page"""
html_template = "index.html"
html_template = ''
html = ''
def __init__(self, nbcourse):
conf = nbcourse.conf
self.html = conf['output_path'] / conf['pages']['home']['name']
conf = nbcourse.conf.copy()
self.html_path = conf['output_path'] / self.html
self.title = conf['title']
self.notebooks = nbcourse.notebooks
self.parent = None
self.variables = conf
self.template_path = conf['template_path']
self.variables = conf
self.notebooks = nbcourse.notebooks
def _render_template(self):
"""Return html rendered from template and variables"""
# Inject variables into html_template
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
self.template_path.as_posix()))
template = env.get_template(self.html_template)
html_out = template.render(self.variables)
with open(self.html_path, 'w') as f:
f.write(html_out)
def render(self):
"""Render html using templating"""
raise NotImplementedError()
class HomePage(Page):
"""A class to handle a homepage to be rendered"""
html_template = "index.html"
html = "index.html"
name = 'home'
parent = None
parent_name = ''
def __init__(self, nbcourse):
super().__init__(nbcourse)
self.chapters = {}
@staticmethod
......@@ -49,19 +79,21 @@ class HomePage:
def _get_variables(self):
"""Set some variables from conf dictionnary and notebooks files"""
logging.basicConfig(level=logging.INFO)
# Get chapters from notebook files
chapters = []
for nbfile in self.notebooks:
chapter = self._get_chapter(nbfile)
chapter['preview_only'] = chapter['number'] in \
self.variables['chapter_preview_only']
try:
chapter['preview_only'] = chapter['number'] in \
self.variables['chapter_preview_only']
except KeyError:
chapter['preview_only'] = False
chapters.append(chapter)
chapters.sort(key=lambda chapter: chapter['number'])
self.variables.update({'chapters': chapters})
log.info("Homepage template variables:")
log.info(pformat(self.variables))
log.debug("Homepage template variables:")
log.debug(pformat(self.variables))
def _render_template(self):
"""Return html rendered from template and variables"""
......@@ -71,7 +103,7 @@ class HomePage:
self.template_path.as_posix()))
template = env.get_template(self.html_template)
html_out = template.render(self.variables)
with open(self.html, 'w') as f:
with open(self.html_path, 'w') as f:
f.write(html_out)
def render(self):
......@@ -80,29 +112,46 @@ class HomePage:
self._render_template()
class MarkdownPage(HomePage):
class MarkdownPage(Page):
html_template = "article.html"
def __init__(self, nbcourse, html, title, src, parent=None):
def __init__(self, nbcourse, src: Path):
super().__init__(nbcourse)
self.html = nbcourse.conf['output_path'] / html
self.title = title
self.src = nbcourse.conf['pages']['path'] / Path(src)
self.parent = parent
def render(self):
self.src = src
self.html = Path(self.src.name).with_suffix('.html')
self.html_path = nbcourse.conf['output_path'] / self.html
with open(self.src, 'r') as md_file:
metadata, self.md_content = frontmatter.parse(md_file.read())
self.title = metadata['title']
self.parent_name = metadata.get('parent', 'home')
self.name = self.src.name
self.parent = None # to be populate later
def get_menu_list(self):
"""Return a list to be displayed as a top menu"""
# At least current page
menu = [(self.title, self.html)]
page = self # Initialize with current page
# Ascend to parent page
while page.parent.parent:
menu.append((page.parent.title, page.parent.html))
page = page.parent
menu.reverse()
return menu
def render(self, parent):
"""Render markdown file into html file adding ToC"""
self.parent = parent
pattern = re.compile(r'^doc\/(.*)\.md$')
def markdown2html():
"""Return html from markdown file"""
with open(self.src, 'r') as f:
text = f.read()
md = markdown.Markdown(extensions=['fenced_code', 'codehilite',
'toc'])
body = md.convert(text)
body = md.convert(self.md_content)
# Transform link to file.md into link to file.html
soup = BeautifulSoup(body, 'html.parser')
......@@ -112,20 +161,9 @@ class MarkdownPage(HomePage):
return soup.prettify(), md.toc
def get_menu_list():
"""Return a list to be displayed as a top menu"""
# At least current page
menu = [(self.title, self.html)]
page = self
while page.parent.parent:
menu.append((page.parent.title, page.parent.html))
page = page.parent
menu.reverse()
return menu
body, toc = markdown2html()
self.variables = {'title': self.title,
'article_content': body,
'toc': toc,
'menu': get_menu_list()}
'menu': self.get_menu_list()}
self._render_template()
../skeleton
\ No newline at end of file
import shutil
from pathlib import Path
from zipfile import ZipFile
import os
import inspect
import re
IGNORED = '__pycache__'
......@@ -14,7 +15,7 @@ def update_material(src: Path, dst: Path):
def get_file_list(path: Path, relative: bool = True, ignore: list = None):
"""Return a list of file paths relative to path"""
ignored = {[IGNORED]} | {ignore}
ignored = set([IGNORED]) | {ignore}
def get_path(obj):
"""return a relative or absolute path"""
......@@ -33,26 +34,43 @@ def update_dict(current: dict, new: dict):
"""Update current multi-level dict with new dict"""
for k, v in new.items():
if k in current and type(v) is dict:
# Update current entry by descending in sub dict
# Update current entry by descending into sub dict
update_dict(current[k], v)
else:
# Add new entry or update with new value
current[k] = v
def zip_files(zip_file_name: Path, paths_to_zip: list, refpath: Path = None,
arcdir: str = None):
"""Archive files_to_zip in zip file"""
def zip_files(zip_file_name: Path, paths_to_zip: list,
ref_to_arc: tuple = None):
"""
Archive files_to_zip in zip file
:param paths_to_zip: a list of files to be archived
:param zip_file_name: the archive filename
:param ref_to_arc: a ('refpath', 'arcdir') tuple
"""
if ref_to_arc:
refpath = ref_to_arc[0]
arcdir = ref_to_arc[1]
with ZipFile(zip_file_name, 'w') as zf:
for path in paths_to_zip:
# Convert refpath/file in arcdir/file if asked
arcpath = arcdir / path.relative_to(refpath) if refpath else None
arcpath = arcdir / path.relative_to(refpath) \
if ref_to_arc else None
zf.write(path, arcname=arcpath)
def clean_file(file):
"""Remove file ignoring errors"""
try:
os.remove(file)
except OSError:
pass
def get_functions(namespace, pattern: str) -> list:
"""
Return the list of functions in namespace that match the regex pattern
"""
all_functions = inspect.getmembers(namespace, inspect.isfunction)
p = re.compile(pattern)
functions = []
for function, dummy in all_functions:
m = p.match(function)
if m:
functions.append(m.group(1))
return functions
......@@ -7,4 +7,5 @@ latex
markdown
rise
pyyaml
doit
\ No newline at end of file
doit
python-frontmatter
\ No newline at end of file
......@@ -47,17 +47,4 @@ license:
icon:
path: img/by-sa.svg
width: 70px
pages:
home:
html: index.html
children:
manual:
html: manual.html
md: manual.md
title: Notice
children:
anaconda:
html: anaconda.html
md: anaconda.md
title: Anaconda
local_reveal: True
---
title: Anaconda
parent: manual.md
---
## Download
[Download Anaconda](https://www.anaconda.com/download) for your operating system and install it.
......
---
title: Manual
parent: home
---
## Execute the Jupyter notebooks
This course content is provided as Jupyter notebooks that require to be engined by a Jupyter server with Python3 kernel.
......
{% extends "base.html" %}
{% block content %}
{% if picture %}
<img src="{{ picture.path }}" alt="{{ picture.path }}" width="{{ picture.width }}">
{% endif %}
{% if subtitle %}<h3>{{ subtitle }}</h3><hr>{% endif %}
<h1>{{ title }}</h1>
<hr>
......
import nbcourse
from nbcourse.initialize import initialize
import contextlib
from pathlib import Path
import os
def test_initialize(tmpdir):
"""Test if initialize function creates the right project skeleton"""
# Get skeleton content
skeleton_path = Path(nbcourse.__path__[0]) / Path("skeleton")
skeleton_paths = set((path.relative_to(skeleton_path)
for path in skeleton_path.glob('**/*')))
# Create a temporary project dir
p = tmpdir.mkdir("test_project")
os.chdir(p)
# Initialize and compare
initialize()
paths = set((path.relative_to(p) for path in Path(p).glob('**/*')))
assert paths == skeleton_paths
from nbcourse.initialize import initialize
from nbcourse.nbcourse import NbCourse, MyDoitMain, ClassTaskLoader
from nbcourse.utils import get_functions
import pytest
from pathlib import Path
import os
from distutils.dir_util import copy_tree
TESTS_PATH = Path(__file__).parent.absolute()
@pytest.fixture
def create_project(tmpdir):
"""Create a sample nbcourse project"""
p = tmpdir.mkdir("test_project")
os.chdir(p)
initialize()
# Create a link to notebooks sample
nb = Path('notebooks')
nb.symlink_to(TESTS_PATH / 'notebooks')
return p
def test_minimal_nbcourse(create_project):
"""Only test nbcourse object instantiation"""
NbCourse()
def test_minimal_nbcourse_build_pages(create_project):
course = NbCourse()
with pytest.raises(SystemExit):
course.build(["build_pages"])
@pytest.mark.parametrize('task', get_functions(NbCourse, 'task_(.*)'))
def test_minimal_nbcourse_build(create_project, task):
"""test nbcourse on list of doit tasks returned by get_functions"""
course = NbCourse()
with pytest.raises(SystemExit):
course.build(["-n 4", task])
from nbcourse.utils import update_dict, get_functions
from nbcourse.nbcourse import NbCourse
import os
from pprint import pprint
import pytest
def test_update_dict():
cur = {'1-a': 1,
'1-b': {'2-a': 2,
'2-b': {'3-a': 3}
}
}
new = {'1-a': 11,
'1-c'