Unitary tests

.gitignore

@@ -8,3 +8,4 @@ __pycache__/
 .env/
 build/
 notebooks/
+tests/notebooks

dev_requirements.txt

+pytest
\ No newline at end of file

nbcourse/__init__.py

@@ -4,20 +4,26 @@ Build a small website to host Jupyter notebooks as course chapters
 
 import argparse
 from .nbcourse import NbCourse, DEFAULT_CONFIG_FILE
-from .quickstart import initialize
+from .initialize import initialize
 from pathlib import Path
 from .mydoit import MyDoit, ClassTaskLoader
 
 
-def main():
-
+def get_help_epilog() -> str:
+    """Instanciate a minimal nbcourse to return help string"""
     course = NbCourse()
     loader = ClassTaskLoader(course)
+    return MyDoit(loader).get_help()
+
+
+def main():
+    """CLI for nbcourse"""
 
     prog = globals()['__package__']
     description = globals()['__doc__']
     formatter = argparse.RawDescriptionHelpFormatter
-    epilog = MyDoit(loader).get_help()
+    epilog = get_help_epilog()
+
     parser = argparse.ArgumentParser(prog=prog,
                                      description=description,
                                      formatter_class=formatter,
@@ -28,7 +34,7 @@ def main():
     generate = parser.add_argument_group('generate')
     generate.add_argument('--config', dest='config_file', type=Path,
                           default=DEFAULT_CONFIG_FILE,
-                          help='Lad YAML file describing site configuration '
+                          help='Load YAML file describing site configuration '
                                '(default: %(default)s)')
 
     # Handle undefined arguments
@@ -38,4 +44,4 @@ def main():
         initialize()
     else:
         course = NbCourse(parsed.config_file)
-        course.run(extra)
+        course.build(extra)

nbcourse/book.py

@@ -27,7 +27,7 @@ class BookContext:
         return self
 
     def __exit__(self, *args):
-        os.remove(self.filepath)
+        self.filepath.unlink()
         os.chdir(self.initial_path)
 
 
@@ -38,7 +38,8 @@ class Book:
 
         self.output_path = conf['output_path']
         self.project_path = conf['project_path']
-        self.titlepage_path = conf['pages']['path'] / conf['pages']['titlepage']
+        self.titlepage_path = conf['pages']['path'] / \
+            conf['pages']['titlepage']
         self.book_title = conf['book_title']
         self.bookbook_filename = conf['bookbook_filename']
         self.template_path = conf['template_path'].as_posix()

nbcourse/quickstart.py → nbcourse/initialize.py

 
-import os
 from distutils.dir_util import copy_tree
 from pathlib import Path
 
@@ -9,4 +8,4 @@ SKEL_PATH = Path(__file__).absolute().parents[1] / Path("skeleton")
 
 def initialize():
     """Initialize a nbcourse directory"""
-    copy_tree(SKEL_PATH, os.getcwd())
+    copy_tree(SKEL_PATH, str(Path.cwd()))

nbcourse/nbcourse.py

@@ -17,7 +17,10 @@ import nbformat
 from nbconvert.preprocessors import ExecutePreprocessor
 from nbconvert import HTMLExporter, SlidesExporter
 from .mydoit import MyDoitMain, ClassTaskLoader
+import logging
 
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
 
 DEFAULT_CONFIG_FILE = "nbcourse.yml"
 
@@ -36,12 +39,21 @@ class NbCourse:
                     'pages': {
                         'dir': 'pages',
                         'titlepage': 'titlepage.tex',
-                        'home': {'name': 'index.html'}
+                        'home': 'index.html'
                         },
                     'bookbook_filename': 'bookbook.tex',
                     'local_reveal': False,
                     'slug_title': 'course',
                     'output_dir': 'build',
+                    'title': 'A course',
+                    'links': [{
+                        'title': 'Manual',
+                        'target': 'manual.html',
+                        'icon': {
+                            'path': 'img/Infobox_info_icon.svg',
+                            'width': '35px',
+                            }
+                        }]
                     }
 
     def __init__(self, config_file: Path = None):
@@ -49,6 +61,9 @@ class NbCourse:
         if config_file:
             update_dict(self.conf, self._get_user_config(config_file))
             self.conf['project_path'] = Path(config_file).absolute().parent
+        else:
+            # Only default config is loaded (only useful for tests)
+            self.conf['project_path'] = Path.cwd()
         self.conf['template_path'] = Path(self.conf['theme']['dir'],
                                           'templates')
         for key in 'nb', 'theme':
@@ -66,7 +81,7 @@ class NbCourse:
             '.pdf')
         self.notebooks = tuple(self.conf['nb']['path'].glob('*-*.ipynb'))
         self.book = self.conf['output_path'] / self.conf['book_title']
-        self.md_pages = list(self.conf['pages']['path'].glob('*.md'))
+        self.md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
         self.html_pages = self._get_pages()  # homepage and documentation pages
         self.theme_files = [file for file in
                             self.conf['theme']['path'].glob('**/*')
@@ -96,12 +111,10 @@ class NbCourse:
     def _get_pages(self):
         """get a list of target html pages from markdown source"""
         # Start html_pages list with index.html
-        html_pages = [self.conf['output_path'] /
-                      self.conf['pages']['home']['name']]
-        for md_page in self.md_pages:
+        html_pages = [self.conf['output_path'] / self.conf['pages']['home']]
+        for path in self.md_page_paths:
             html_pages.append(self.conf['output_path'] /
-                              md_page.relative_to(
-                              self.conf['pages']['path']).with_suffix('.html'))
+                              Path(path.name).with_suffix('.html'))
         return html_pages
 
     @staticmethod
@@ -117,28 +130,27 @@ class NbCourse:
         Build a mini website using html templating and markdown conversion
         """
 
-        def render_children(parent, parent_page):
+        # List mardkown files
+        md_page_paths = list(self.conf['pages']['path'].glob('*.md'))
+        # Instanciate all markdown pages to be rendered
+        md_pages = [MarkdownPage(self, md_page_path)
+                    for md_page_path in md_page_paths]
+
+        def render_children(parent):
             """Recursive function to render children pages from parent page"""
-            try:
-                children = parent['children']
-                for child in children.values():
-                    # Render markdown pages
-                    child_page = MarkdownPage(self, child['html'],
-                                              title=child['title'],
-                                              src=child['md'],
-                                              parent=parent_page)
-                    child_page.render()
-                    render_children(child, child_page)
-            except KeyError:
-                # Parent page has no children
-                return
+            children = (md_page for md_page in md_pages
+                        if md_page.parent_name == parent.name)
+            for child in children:
+                # Render markdown pages
+                child.render(parent)
+                render_children(child)
 
         # First build homepage
         homepage = HomePage(self)
         homepage.render()
 
         # Then build children pages
-        render_children(self.conf['pages']['home'], homepage)
+        render_children(homepage)
 
     def build_book(self):
         """Build a pdf book using bookbook"""
@@ -189,6 +201,11 @@ class NbCourse:
                 'actions': [(update_material, (src_path, dst_path))],
                 'clean': [(clean_tree, (dst_path,))],
             }
+        else:
+            return {
+                'uptodate': [True],
+                'actions': []
+            }
 
     def execute_notebook(self, notebook: Path, executed_notebook: Path):
         """Execute notebook and write result to executed_notebook file"""
@@ -214,6 +231,7 @@ class NbCourse:
             yield {
                 'name': executed_notebook,
                 'file_dep': [notebook],
+                'task_dep': ['copy_material'],
                 'targets': [executed_notebook],
                 'clean': True,
                 'actions': [(self.execute_notebook, (notebook,
@@ -268,7 +286,7 @@ class NbCourse:
             }
 
     def task_zip_chapters(self):
-        """Build one zip archive per chapter"""
+        """Build zip archives for single chapter downloads"""
 
         for notebook in self.notebooks:
             zip_file = self.get_executed_notebook(notebook).with_suffix('.zip')
@@ -279,6 +297,7 @@ class NbCourse:
             yield {
                 'name': zip_file,
                 'file_dep': paths_to_zip,
+                'task_dep': ['copy_material'],
                 'targets': [zip_file],
                 'clean': True,
                 'actions': [(zip_files, (zip_file, paths_to_zip))],
@@ -287,7 +306,8 @@ class NbCourse:
     def task_build_pages(self):
         """Build html pages"""
         return {
-            'file_dep': self.md_pages + self.theme_files,
+            'file_dep': self.md_page_paths + self.theme_files,
+            'task_dep': ['copy_material'],
             'targets': self.html_pages,
             'clean': True,
             'actions': [self.build_pages],
@@ -311,11 +331,14 @@ class NbCourse:
             'targets': [self.zip_file],
             'clean': True,
             'actions': [(zip_files, (self.zip_file, paths_to_zip),
-                         {'refpath': self.conf['output_path'],
-                          'arcdir': self.conf['slug_title']})],
+                         {'ref_to_arc': (self.conf['output_path'],
+                                         self.conf['slug_title'])})],
         }
 
-    def run(self, args: list):
+    def build(self, args: list = None):
+        """Build course using Doit"""
+        if args is None:
+            args = []
         doit_config = {}  # TODO: implement verbosity option
         sys.exit(MyDoitMain(ClassTaskLoader(self),
                             extra_config=doit_config).run(args))

nbcourse/pages.py

@@ -6,23 +6,53 @@ from pathlib import Path
 from pprint import pformat
 import jinja2
 from bs4 import BeautifulSoup
+import frontmatter
 
 log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
 
 
-class HomePage:
-    """A class to handle a homepage to be rendered"""
+class Page:
+    """An abstract class for a static web page"""
 
-    html_template = "index.html"
+    html_template = ''
+    html = ''
 
     def __init__(self, nbcourse):
-        conf = nbcourse.conf
-        self.html = conf['output_path'] / conf['pages']['home']['name']
+        conf = nbcourse.conf.copy()
+        self.html_path = conf['output_path'] / self.html
         self.title = conf['title']
-        self.notebooks = nbcourse.notebooks
-        self.parent = None
-        self.variables = conf
         self.template_path = conf['template_path']
+        self.variables = conf
+        self.notebooks = nbcourse.notebooks
+
+    def _render_template(self):
+        """Return html rendered from template and variables"""
+
+        # Inject variables into html_template
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(
+                                 self.template_path.as_posix()))
+        template = env.get_template(self.html_template)
+        html_out = template.render(self.variables)
+        with open(self.html_path, 'w') as f:
+            f.write(html_out)
+
+    def render(self):
+        """Render html using templating"""
+        raise NotImplementedError()
+
+
+class HomePage(Page):
+    """A class to handle a homepage to be rendered"""
+
+    html_template = "index.html"
+    html = "index.html"
+    name = 'home'
+    parent = None
+    parent_name = ''
+
+    def __init__(self, nbcourse):
+        super().__init__(nbcourse)
         self.chapters = {}
 
     @staticmethod
@@ -49,19 +79,21 @@ class HomePage:
 
     def _get_variables(self):
         """Set some variables from conf dictionnary and notebooks files"""
-        logging.basicConfig(level=logging.INFO)
 
         # Get chapters from notebook files
         chapters = []
         for nbfile in self.notebooks:
             chapter = self._get_chapter(nbfile)
-            chapter['preview_only'] = chapter['number'] in \
-                self.variables['chapter_preview_only']
+            try:
+                chapter['preview_only'] = chapter['number'] in \
+                    self.variables['chapter_preview_only']
+            except KeyError:
+                chapter['preview_only'] = False
             chapters.append(chapter)
         chapters.sort(key=lambda chapter: chapter['number'])
         self.variables.update({'chapters': chapters})
-        log.info("Homepage template variables:")
-        log.info(pformat(self.variables))
+        log.debug("Homepage template variables:")
+        log.debug(pformat(self.variables))
 
     def _render_template(self):
         """Return html rendered from template and variables"""
@@ -71,7 +103,7 @@ class HomePage:
                                  self.template_path.as_posix()))
         template = env.get_template(self.html_template)
         html_out = template.render(self.variables)
-        with open(self.html, 'w') as f:
+        with open(self.html_path, 'w') as f:
             f.write(html_out)
 
     def render(self):
@@ -80,29 +112,46 @@ class HomePage:
         self._render_template()
 
 
-class MarkdownPage(HomePage):
+class MarkdownPage(Page):
 
     html_template = "article.html"
 
-    def __init__(self, nbcourse, html, title, src, parent=None):
+    def __init__(self, nbcourse, src: Path):
         super().__init__(nbcourse)
-        self.html = nbcourse.conf['output_path'] / html
-        self.title = title
-        self.src = nbcourse.conf['pages']['path'] / Path(src)
-        self.parent = parent
-
-    def render(self):
+        self.src = src
+        self.html = Path(self.src.name).with_suffix('.html')
+        self.html_path = nbcourse.conf['output_path'] / self.html
+
+        with open(self.src, 'r') as md_file:
+            metadata, self.md_content = frontmatter.parse(md_file.read())
+        self.title = metadata['title']
+        self.parent_name = metadata.get('parent', 'home')
+        self.name = self.src.name
+        self.parent = None  # to be populate later
+
+    def get_menu_list(self):
+        """Return a list to be displayed as a top menu"""
+        # At least current page
+        menu = [(self.title, self.html)]
+        page = self  # Initialize with current page
+        # Ascend to parent page
+        while page.parent.parent:
+            menu.append((page.parent.title, page.parent.html))
+            page = page.parent
+        menu.reverse()
+        return menu
+
+    def render(self, parent):
         """Render markdown file into html file adding ToC"""
 
+        self.parent = parent
         pattern = re.compile(r'^doc\/(.*)\.md$')
 
         def markdown2html():
             """Return html from markdown file"""
-            with open(self.src, 'r') as f:
-                text = f.read()
             md = markdown.Markdown(extensions=['fenced_code', 'codehilite',
                                                'toc'])
-            body = md.convert(text)
+            body = md.convert(self.md_content)
 
             # Transform link to file.md into link to file.html
             soup = BeautifulSoup(body, 'html.parser')
@@ -112,20 +161,9 @@ class MarkdownPage(HomePage):
 
             return soup.prettify(), md.toc
 
-        def get_menu_list():
-            """Return a list to be displayed as a top menu"""
-            # At least current page
-            menu = [(self.title, self.html)]
-            page = self
-            while page.parent.parent:
-                menu.append((page.parent.title, page.parent.html))
-                page = page.parent
-            menu.reverse()
-            return menu
-
         body, toc = markdown2html()
         self.variables = {'title': self.title,
                           'article_content': body,
                           'toc': toc,
-                          'menu': get_menu_list()}
+                          'menu': self.get_menu_list()}
         self._render_template()

nbcourse/skeleton

+../skeleton
\ No newline at end of file

nbcourse/utils.py

 import shutil
 from pathlib import Path
 from zipfile import ZipFile
-import os
+import inspect
+import re
 
 IGNORED = '__pycache__'
 
@@ -14,7 +15,7 @@ def update_material(src: Path, dst: Path):
 
 def get_file_list(path: Path, relative: bool = True, ignore: list = None):
     """Return a list of file paths relative to path"""
-    ignored = {[IGNORED]} | {ignore}
+    ignored = set([IGNORED]) | {ignore}
 
     def get_path(obj):
         """return a relative or absolute path"""
@@ -33,26 +34,43 @@ def update_dict(current: dict, new: dict):
     """Update current multi-level dict with new dict"""
     for k, v in new.items():
         if k in current and type(v) is dict:
-            # Update current entry by descending in sub dict
+            # Update current entry by descending into sub dict
             update_dict(current[k], v)
         else:
             # Add new entry or update with new value
             current[k] = v
 
 
-def zip_files(zip_file_name: Path, paths_to_zip: list, refpath: Path = None,
-              arcdir: str = None):
-    """Archive files_to_zip in zip file"""
+def zip_files(zip_file_name: Path, paths_to_zip: list,
+              ref_to_arc: tuple = None):
+    """
+    Archive files_to_zip in zip file
+
+    :param paths_to_zip: a list of files to be archived
+    :param zip_file_name: the archive filename
+    :param ref_to_arc: a ('refpath', 'arcdir') tuple
+    """
+    if ref_to_arc:
+        refpath = ref_to_arc[0]
+        arcdir = ref_to_arc[1]
+
     with ZipFile(zip_file_name, 'w') as zf:
         for path in paths_to_zip:
             # Convert refpath/file in arcdir/file if asked
-            arcpath = arcdir / path.relative_to(refpath) if refpath else None
+            arcpath = arcdir / path.relative_to(refpath) \
+                      if ref_to_arc else None
             zf.write(path, arcname=arcpath)
 
 
-def clean_file(file):
-    """Remove file ignoring errors"""
-    try:
-        os.remove(file)
-    except OSError:
-        pass
+def get_functions(namespace, pattern: str) -> list:
+    """
+    Return the list of functions in namespace that match the regex pattern
+    """
+    all_functions = inspect.getmembers(namespace, inspect.isfunction)
+    p = re.compile(pattern)
+    functions = []
+    for function, dummy in all_functions:
+        m = p.match(function)
+        if m:
+            functions.append(m.group(1))
+    return functions

requirements.txt

@@ -7,4 +7,5 @@ latex
 markdown
 rise
 pyyaml
-doit
\ No newline at end of file
+doit
+python-frontmatter
\ No newline at end of file

skeleton/nbcourse.yml

@@ -47,17 +47,4 @@ license:
   icon:
     path: img/by-sa.svg
     width: 70px
-pages:
-  home:
-    html: index.html
-    children:
-      manual:
-        html: manual.html
-        md: manual.md
-        title: Notice
-        children:
-          anaconda:
-            html: anaconda.html
-            md: anaconda.md
-            title: Anaconda
 local_reveal: True

skeleton/pages/anaconda.md

+---
+title: Anaconda
+parent: manual.md
+---
+
 ## Download
 
 [Download Anaconda](https://www.anaconda.com/download) for your operating system and install it.

skeleton/pages/manual.md

+---
+title: Manual
+parent: home
+---
+
 ## Execute the Jupyter notebooks
 
 This course content is provided as Jupyter notebooks that require to be engined by a Jupyter server with Python3 kernel.

skeleton/theme/default/templates/index.html

 {% extends "base.html" %}
 
         {% block content %}
+        {% if picture %}
         <img src="{{ picture.path }}" alt="{{ picture.path }}" width="{{ picture.width }}">
+        {% endif %}
         {% if subtitle %}<h3>{{ subtitle }}</h3><hr>{% endif %}
         <h1>{{ title }}</h1>
         <hr>

tests/test_initialize.py

+import nbcourse
+from nbcourse.initialize import initialize
+import contextlib
+from pathlib import Path
+import os
+
+
+def test_initialize(tmpdir):
+    """Test if initialize function creates the right project skeleton"""
+    # Get skeleton content
+    skeleton_path = Path(nbcourse.__path__[0]) / Path("skeleton")
+    skeleton_paths = set((path.relative_to(skeleton_path)
+                          for path in skeleton_path.glob('**/*')))
+
+    # Create a temporary project dir
+    p = tmpdir.mkdir("test_project")
+    os.chdir(p)
+
+    # Initialize and compare
+    initialize()
+    paths = set((path.relative_to(p) for path in Path(p).glob('**/*')))
+    assert paths == skeleton_paths

tests/test_nbcourse.py

+from nbcourse.initialize import initialize
+from nbcourse.nbcourse import NbCourse, MyDoitMain, ClassTaskLoader
+from nbcourse.utils import get_functions
+import pytest
+from pathlib import Path
+import os
+from distutils.dir_util import copy_tree
+
+TESTS_PATH = Path(__file__).parent.absolute()
+
+
+@pytest.fixture
+def create_project(tmpdir):
+    """Create a sample nbcourse project"""
+    p = tmpdir.mkdir("test_project")
+    os.chdir(p)
+    initialize()
+
+    # Create a link to notebooks sample
+    nb = Path('notebooks')
+    nb.symlink_to(TESTS_PATH / 'notebooks')
+
+    return p
+
+
+def test_minimal_nbcourse(create_project):
+    """Only test nbcourse object instantiation"""
+    NbCourse()
+
+
+def test_minimal_nbcourse_build_pages(create_project):
+    course = NbCourse()
+    with pytest.raises(SystemExit):
+        course.build(["build_pages"])
+
+
+@pytest.mark.parametrize('task', get_functions(NbCourse, 'task_(.*)'))
+def test_minimal_nbcourse_build(create_project, task):
+    """test nbcourse on list of doit tasks returned by get_functions"""
+    course = NbCourse()
+    with pytest.raises(SystemExit):
+        course.build(["-n 4", task])

tests/test_utils.py

+from nbcourse.utils import update_dict, get_functions
+from nbcourse.nbcourse import NbCourse
+import os
+from pprint import pprint
+import pytest
+
+
+def test_update_dict():
+
+    cur = {'1-a': 1,
+           '1-b': {'2-a': 2,
+                   '2-b': {'3-a': 3}
+                   }
+           }
+    new = {'1-a': 11,
+           '1-c'