PKcDO43tmaestro/__init__.py""" Maestro is a collection of tools to aid teachers in programming-based classrooms. """ __version__ = '0.1.0' __name__ = 'maestro' __author__ = 'Fábio Mendes 'PKNCOmaestro/notebook.pyPKtPCOs{O  maestro/widgets.py""" Useful widgets to be embedded in a Jupyter notebook. """ from IPython.display import display from hyperpython import iframe, h def display_youtube(video_id, **kwargs): """ Displays the youtube video with the given video id. It accepts all arguments of :func:`show_layout` and :func:`youtube`. """ layout_kwargs, kwargs = split_layout_args(kwargs, ('width', 'height')) display_layout(youtube(video_id, **kwargs), **layout_kwargs) def display_layout(element, **kwargs): """ Show element inside layout. See :func:`layout` for more information. """ display(layout(element, **kwargs)) # # COMPONENTS # # Functions in this section return Hyperpython elements instead of displaying # them using IPython's display function. # def layout(element, center=False): """ Layouts the given element according to options. Args: element: An hyperpython element. center (bool): If True, centralizes the element. """ if center: element = h('center', element) return element def youtube(video_id, class_=None, width=640, heigth=480): """ Return an Hyperpython element with an youtube video iframe. Args: video_id: The unique youtube video identifier. """ return iframe(class_=class_, width=width, height=heigth, src='https://www.youtube.com/embed/' + video_id, allow="accelerometer; autoplay; encrypted-media; gyroscope; " "picture-in-picture", allowfullscreen=True, frameborder=0, ) # # UTILITIES # def split_layout_args(kwargs, skip=()): """ Split a dictionary of arguments into arguments that should be applied to layout from arguments that should be passed forward. Args: kwargs (mapping): A mapping of names to values skip (sequence): A optional set of arguments that should not be assigned to layout arguments. Returns: A tuple of (layout_args, non_layout_args) dictionaries. """ layout_args = {'center'} layout_args.difference_update(skip) in_args = {k: v for k, v in kwargs.items() if k in layout_args} out_args = {k: v for k, v in kwargs.items() if k not in layout_args} return in_args, out_args PK'NP 33 maestro/classroom_db/__init__.pyfrom .classroom import Classroom, Student, StudentsPK7NVF$!maestro/classroom_db/classroom.pyimport datetime as dt import json import os from typing import List, Optional import toml from unidecode import unidecode from maestro.classroom_db.queryable import Queryable from sidekick import Record, lazy class Student(Record): """ A student with meta information. """ name: str school_id: str = None email: str = None aliases: List[str] = lazy(lambda _: []) @property def display(self): display = self.name if self.school_id: display += f' ({self.school_id})' return display def __init__(self, name: str, school_id=None, email=None, aliases=()): assert isinstance(name, str), name super().__setattr__('name', name) super().__setattr__('school_id', school_id) super().__setattr__('email', email) super().__setattr__('aliases', list(aliases)) def __repr__(self): data = repr(self.name) if self.school_id: data = f'{data}, {self.school_id}' if self.email: data = f'{data}, email={self.email!r}' if self.aliases: data = f'{data}, aliases={self.aliases!r}' return f'{self.__class__.__name__}({data})' # Derived properties @property def normalized_name(self): name = self.name.strip(' \t\n\r\f\'"-/:_()[]{}') return unidecode(name.lower().replace(' ', '_')) @classmethod def from_json(cls, data): return cls(**data) def to_json(self, nbgrader=False): if nbgrader: first_name, *rest = self.name.split() last_name = ' '.join(rest) return { 'id': self.school_id, 'first_name': first_name, 'last_name': last_name, 'email': self.email, } return { 'name': self.name, 'school_id': self.school_id, 'email': self.email, 'aliases': self.aliases, } class Students(Queryable): """ A collection of students. Can query, modify and persist results. """ @classmethod def from_json(cls, data): if data is None: return cls() return cls(map(Student.from_json, data)) def to_json(self): return [x.to_json() for x in self.data] def add(self, student): """ Adds student in database. """ if student.school_id is None: raise ValueError('cannot add element with null school_id') school_id = student.school_id if any(s for s in self.data if s.school_id == school_id): raise ValueError('Student with given id already exists on database') self.data.append(student) def by_alias(self, alias) -> Student: """ Return student by alias. """ return self.filter_by_alias(alias).single() def by_pk(self, pk): return self.filter(lambda x: x['school_id'] == pk).single() def filter_by_alias(self, alias, inplace=False) -> 'Students': """ Return elements with the given alias. """ gen = (s for s in self.data if alias in s.aliases) return self._from_generator(gen, inplace) def best_match(self, student) -> Optional[Student]: """ Return the best match for the given student object or None if no good match was found. """ best = [] school_id = student.school_id or object() ref = student.normalized_name for std in self.data: if std.school_id == school_id: return std if std.normalized_name == ref: best.append((std, 10)) best.sort(key=lambda x: x[1], reverse=True) if best: return best[0][0] return None class Classroom: """ Describes a classroom. """ name: str = None created: dt.datetime teacher: str = None students: Students = None assignments: list = None path: str = None @classmethod def load(cls, path) -> 'Classroom': """ Load classroom from toml or json file. """ try: with open(path) as fd: mod = get_db_module(path) data = mod.load(fd) except FileNotFoundError: data = {} out = cls.from_json(data) out.path = path return out @classmethod def from_json(cls, data) -> 'Classroom': """ Initialize classroom from JSON-like data created with the to_json() method. """ info = data.get('classroom', {}) out = cls(info.get('name', 'classroom'), info.get('teacher', 'teacher'), assignments=info.get('assignments', [])) out.students = Students.from_json(data.get('students')) out.created = info.get('created', dt.datetime.now()) return out def __init__(self, name, teacher, students=(), assignments=(), **kwargs): self.name = name self.teacher = teacher self.students = Students(students) self.created = dt.datetime.now() self.assignments = list(assignments) for k, v in kwargs.items(): if hasattr(self, k): setattr(self, k, v) else: raise TypeError(f'invalid argument: {k}') def to_json(self): return { 'classroom': { 'name': self.name, 'teacher': self.teacher, 'created': self.created, 'assignments': self.assignments, }, 'students': self.students.to_json() } def save(self, path=None): path = path or self.path if not path: raise ValueError('path must be specified.') with open(path, 'w') as fd: mod = get_db_module(path) mod.dump(self.to_json(), fd) return self def get_db_module(path): ext = os.path.splitext(path)[-1] if ext == '.json': return json elif ext == '.toml': return toml else: raise ValueError(f'invalid extension: {ext}') PKN-xz!maestro/classroom_db/nb_config.pyfrom .classroom import Classroom EXAMPLE = """ from maestro.classroom_db.nb_config import nb_config c = get_config() nb_config(c) """ def nb_config(cfg): """ Replaces nbgrader_config.py. Based on "nbgrader quickstart" """ maestro = Classroom.load('config.toml') # You only need this if you are running nbgrader on a shared # server set up. # cfg.Exchange.course_id = maestro.name # cfg.Exchange.root = "/tmp/nb-exchange" # Update this list with other assignments you want cfg.CourseDirectory.db_students = [s.to_json(nbgrader=True) for s in maestro.students] # Change the students in this list with that actual students in # your course cfg.CourseDirectory.db_assignments = [{'name': name} for name in maestro.assignments] cfg.IncludeHeaderFooter.header = "source/header.ipynb" ############################################################################### # End additions by nbgrader quickstart ############################################################################### # Configuration file for nbgrader. # ------------------------------------------------------------------------------ # Application(SingletonConfigurable) configuration # ------------------------------------------------------------------------------ # This is an application. # The date format used by logging formatters for %(asctime)s # c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S' # The Logging format template # c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s' # Set the log level by value or name. # c.Application.log_level = 30 # ------------------------------------------------------------------------------ # JupyterApp(Application) configuration # ------------------------------------------------------------------------------ # Base class for Jupyter applications # Answer yes to any prompts. # c.JupyterApp.answer_yes = False # Full path of a config file. # c.JupyterApp.config_file = '' # Specify a config file to load. # c.JupyterApp.config_file_name = '' # Generate default config file. # c.JupyterApp.generate_config = False # ------------------------------------------------------------------------------ # NbGrader(JupyterApp) configuration # ------------------------------------------------------------------------------ # A base class for all the nbgrader apps. # Name of the logfile to log to. # c.NbGrader.logfile = '.nbgrader.log' # ------------------------------------------------------------------------------ # NbGraderApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # CourseDirectory(LoggingConfigurable) configuration # ------------------------------------------------------------------------------ # The assignment name. This MUST be specified, either by setting the config # option, passing an argument on the command line, or using the --assignment # option on the command line. # c.CourseDirectory.assignment_id = '' # The name of the directory that contains assignment submissions after they have # been autograded. This corresponds to the `nbgrader_step` variable in the # `directory_structure` config option. # c.CourseDirectory.autograded_directory = 'autograded' # A list of assignments that will be created in the database. Each item in the # list should be a dictionary with the following keys: # # - name # - duedate (optional) # # The values will be stored in the database. Please see the API documentation on # the `Assignment` database model for details on these fields. # c.CourseDirectory.db_assignments = [] # A list of student that will be created in the database. Each item in the list # should be a dictionary with the following keys: # # - id # - first_name (optional) # - last_name (optional) # - email (optional) # # The values will be stored in the database. Please see the API documentation on # the `Student` database model for details on these fields. # c.CourseDirectory.db_students = [] # URL to the database. Defaults to sqlite:////gradebook.db, where # is another configurable variable. # c.CourseDirectory.db_url = '' # Format string for the directory structure that nbgrader works over during the # grading process. This MUST contain named keys for 'nbgrader_step', # 'student_id', and 'assignment_id'. It SHOULD NOT contain a key for # 'notebook_id', as this will be automatically joined with the rest of the path. # c.CourseDirectory.directory_structure = '{nbgrader_step}/{student_id}/{assignment_id}' # The name of the directory that contains assignment feedback after grading has # been completed. This corresponds to the `nbgrader_step` variable in the # `directory_structure` config option. # c.CourseDirectory.feedback_directory = 'feedback' # List of file names or file globs to be ignored when copying directories. # c.CourseDirectory.ignore = ['.ipynb_checkpoints', '*.pyc', '__pycache__'] # File glob to match notebook names, excluding the '.ipynb' extension. This can # be changed to filter by notebook. # c.CourseDirectory.notebook_id = '*' # The name of the directory that contains the version of the assignment that # will be released to students. This corresponds to the `nbgrader_step` variable # in the `directory_structure` config option. # c.CourseDirectory.release_directory = 'release' # The root directory for the course files (that includes the `source`, # `release`, `submitted`, `autograded`, etc. directories). Defaults to the # current working directory. # c.CourseDirectory.root = '' # The name of the directory that contains the master/instructor version of # assignments. This corresponds to the `nbgrader_step` variable in the # `directory_structure` config option. # c.CourseDirectory.source_directory = 'source' # File glob to match student IDs. This can be changed to filter by student. # Note: this is always changed to '.' when running `nbgrader assign`, as the # assign step doesn't have any student ID associated with it. # # If the ID is purely numeric and you are passing it as a flag on the command # line, you will need to escape the quotes in order to have it detected as a # string, for example `--student=""12345""`. See: # # https://github.com/jupyter/nbgrader/issues/743 # # for more details. # c.CourseDirectory.student_id = '*' # The name of the directory that contains assignments that have been submitted # by students for grading. This corresponds to the `nbgrader_step` variable in # the `directory_structure` config option. # c.CourseDirectory.submitted_directory = 'submitted' # ------------------------------------------------------------------------------ # AssignApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # AutogradeApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # FormgradeApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # FeedbackApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # ValidateApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # ReleaseApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # CollectApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # ZipCollectApp(NbGrader) configuration # ------------------------------------------------------------------------------ # The name of the directory that contains assignment submission files and/or # archives (zip) files manually downloaded from a LMS. This corresponds to the # `collect_step` variable in the `collect_structure` config option. # c.ZipCollectApp.archive_directory = 'archive' # Format string for the directory structure that nbgrader works over during the # zip collect process. This MUST contain named keys for 'downloaded', # 'assignment_id', and 'collect_step'. # c.ZipCollectApp.collect_directory_structure = '{downloaded}/{assignment_id}/{collect_step}' # The plugin class for processing the submitted file names after they have been # extracted into the `extracted_directory`. # c.ZipCollectApp.collector_plugin = 'nbgrader.plugins.zipcollect.FileNameCollectorPlugin' # The main directory that corresponds to the `downloaded` variable in the # `collect_structure` config option. # c.ZipCollectApp.downloaded_directory = 'downloaded' # The name of the directory that contains assignment submission files extracted # or copied from the `archive_directory`. This corresponds to the `collect_step` # variable in the `collect_structure` config option. # c.ZipCollectApp.extracted_directory = 'extracted' # The plugin class for extracting the archive files in the `archive_directory`. # c.ZipCollectApp.extractor_plugin = 'nbgrader.plugins.zipcollect.ExtractorPlugin' # Force overwrite of existing files. # c.ZipCollectApp.force = False # Skip submitted notebooks with invalid names. # c.ZipCollectApp.strict = False # ------------------------------------------------------------------------------ # FetchApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # SubmitApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # ListApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # ExtensionApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # QuickStartApp(NbGrader) configuration # ------------------------------------------------------------------------------ # Whether to overwrite existing files # c.QuickStartApp.force = False # ------------------------------------------------------------------------------ # ExportApp(NbGrader) configuration # ------------------------------------------------------------------------------ # The plugin class for exporting the grades. # c.ExportApp.plugin_class = 'nbgrader.plugins.export.CsvExportPlugin' # ------------------------------------------------------------------------------ # DbApp(NbGrader) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # UpdateApp(NbGrader) configuration # ------------------------------------------------------------------------------ # whether to validate metadata after updating it # c.UpdateApp.validate = True # ------------------------------------------------------------------------------ # ExportPlugin(BasePlugin) configuration # ------------------------------------------------------------------------------ # Base class for export plugins. # destination to export to # c.ExportPlugin.to = '' # ------------------------------------------------------------------------------ # CsvExportPlugin(ExportPlugin) configuration # ------------------------------------------------------------------------------ # CSV exporter plugin. # ------------------------------------------------------------------------------ # ExtractorPlugin(BasePlugin) configuration # ------------------------------------------------------------------------------ # Submission archive files extractor plugin for the # :class:`~nbgrader.apps.zipcollectapp.ZipCollectApp`. Extractor plugin # subclasses MUST inherit from this class. # Force overwrite of existing files. # c.ExtractorPlugin.force = False # List of valid archive (zip) filename extensions to extract. Any archive (zip) # files with an extension not in this list are copied to the # `extracted_directory`. # c.ExtractorPlugin.zip_ext = ['.zip', '.gz'] # ------------------------------------------------------------------------------ # FileNameCollectorPlugin(BasePlugin) configuration # ------------------------------------------------------------------------------ # Submission filename collector plugin for the # :class:`~nbgrader.apps.zipcollectapp.ZipCollectApp`. Collect plugin subclasses # MUST inherit from this class. # This regular expression is applied to each submission filename and MUST be # supplied by the instructor. This regular expression MUST provide the # `(?P...)` and `(?P...)` named group expressions. # Optionally this regular expression can also provide the `(?P...)`, # `(?P...)`, `(?P...)`, and `(?P...)` named group # expressions. For example if the filename is: # # `ps1_bitdiddle_attempt_2016-01-30-15-00-00_problem1.ipynb` # # then this `named_regexp` could be: # # ".*_(?P\w+)_attempt_(?P[0-9\-]+)_(?P\w+)" # # For named group regular expression examples see # https://docs.python.org/howto/regex.html # c.FileNameCollectorPlugin.named_regexp = '' # List of valid submission filename extensions to collect. Any submitted file # with an extension not in this list is skipped. # c.FileNameCollectorPlugin.valid_ext = ['.ipynb'] # ------------------------------------------------------------------------------ # LateSubmissionPlugin(BasePlugin) configuration # ------------------------------------------------------------------------------ # Predefined methods for assigning penalties for late submission # The method for assigning late submission penalties: # 'none': do nothing (no penalty assigned) # 'zero': assign an overall score of zero (penalty = score) # c.LateSubmissionPlugin.penalty_method = 'none' # ------------------------------------------------------------------------------ # NbConvertBase(LoggingConfigurable) configuration # ------------------------------------------------------------------------------ # Global configurable class for shared config # # Useful for display data priority that might be used by many transformers # Deprecated default highlight language as of 5.0, please use language_info # metadata instead # c.NbConvertBase.default_language = 'ipython' # An ordered list of preferred output type, the first encountered will usually # be used when converting discarding the others. # c.NbConvertBase.display_data_priority = ['text/html', 'application/pdf', 'text/latex', 'image/svg+xml', 'image/png', 'image/jpeg', 'text/markdown', 'text/plain'] # ------------------------------------------------------------------------------ # Preprocessor(NbConvertBase) configuration # ------------------------------------------------------------------------------ # A configurable preprocessor # # Inherit from this class if you wish to have configurability for your # preprocessor. # # Any configurable traitlets this class exposed will be configurable in profiles # using c.SubClassName.attribute = value # # you can overwrite :meth:`preprocess_cell` to apply a transformation # independently on each cell or :meth:`preprocess` if you prefer your own logic. # See corresponding docstring for information. # # Disabled by default and can be enabled via the config by # 'c.YourPreprocessorName.enabled = True' ## # c.Preprocessor.enabled = False # ------------------------------------------------------------------------------ # NbGraderPreprocessor(Preprocessor) configuration # ------------------------------------------------------------------------------ # Whether to use this preprocessor when running nbgrader # c.NbGraderPreprocessor.enabled = True # ------------------------------------------------------------------------------ # AssignLatePenalties(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # Preprocessor for assigning penalties for late submissions to the database # The plugin class for assigning the late penalty for each notebook. # c.AssignLatePenalties.plugin_class = 'nbgrader.plugins.latesubmission.LateSubmissionPlugin' # ------------------------------------------------------------------------------ # IncludeHeaderFooter(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor for adding header and/or footer cells to a notebook. # Path to footer notebook, relative to the root of the course directory # c.IncludeHeaderFooter.footer = '' # Path to header notebook, relative to the root of the course directory # c.IncludeHeaderFooter.header = '' # ------------------------------------------------------------------------------ # LockCells(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor for making cells undeletable. # Whether all assignment cells are locked (non-deletable and non-editable) # c.LockCells.lock_all_cells = False # Whether grade cells are locked (non-deletable) # c.LockCells.lock_grade_cells = True # Whether readonly cells are locked (non-deletable and non-editable) # c.LockCells.lock_readonly_cells = True # Whether solution cells are locked (non-deletable and non-editable) # c.LockCells.lock_solution_cells = True # ------------------------------------------------------------------------------ # ClearSolutions(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # The delimiter marking the beginning of a solution # c.ClearSolutions.begin_solution_delimeter = 'BEGIN SOLUTION' # The code snippet that will replace code solutions cfg.ClearSolutions.code_stub = { 'python': 'NotImplemented # escreva sua solução aqui...'} # The delimiter marking the end of a solution # c.ClearSolutions.end_solution_delimeter = 'END SOLUTION' # Whether or not to complain if cells containing solutions regions are not # marked as solution cells. WARNING: this will potentially cause things to break # if you are using the full nbgrader pipeline. ONLY disable this option if you # are only ever planning to use nbgrader assign. # c.ClearSolutions.enforce_metadata = True # The text snippet that will replace written solutions cfg.ClearSolutions.text_stub = 'ESCREVA SUA SOLUÇÃO' # ------------------------------------------------------------------------------ # SaveAutoGrades(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # Preprocessor for saving out the autograder grades into a database # ------------------------------------------------------------------------------ # ComputeChecksums(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor to compute checksums of grade cells. # ------------------------------------------------------------------------------ # SaveCells(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor to save information about grade and solution cells. # ------------------------------------------------------------------------------ # OverwriteCells(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor to overwrite information about grade and solution cells. # ------------------------------------------------------------------------------ # CheckCellMetadata(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor for checking that grade ids are unique. # ------------------------------------------------------------------------------ # ExecutePreprocessor(Preprocessor) configuration # ------------------------------------------------------------------------------ # Executes all the cells in a notebook # If `False` (default), when a cell raises an error the execution is stopped and # a `CellExecutionError` is raised. If `True`, execution errors are ignored and # the execution is continued until the end of the notebook. Output from # exceptions is included in the cell output in both cases. # c.ExecutePreprocessor.allow_errors = False # If False (default), errors from executing the notebook can be allowed with a # `raises-exception` tag on a single cell, or the `allow_errors` configurable # option for all cells. An allowed error will be recorded in notebook output, # and execution will continue. If an error occurs when it is not explicitly # allowed, a `CellExecutionError` will be raised. If True, `CellExecutionError` # will be raised for any error that occurs while executing the notebook. This # overrides both the `allow_errors` option and the `raises-exception` cell tag. # c.ExecutePreprocessor.force_raise_errors = False # If execution of a cell times out, interrupt the kernel and continue executing # other cells rather than throwing an error and stopping. # c.ExecutePreprocessor.interrupt_on_timeout = False # The time to wait (in seconds) for IOPub output. This generally doesn't need to # be set, but on some slow networks (such as CI systems) the default timeout # might not be long enough to get all messages. # c.ExecutePreprocessor.iopub_timeout = 4 # The kernel manager class to use. # c.ExecutePreprocessor.kernel_manager_class = 'builtins.object' # Name of kernel to use to execute the cells. If not set, use the kernel_spec # embedded in the notebook. # c.ExecutePreprocessor.kernel_name = '' # If `False` (default), then the kernel will continue waiting for iopub messages # until it receives a kernel idle message, or until a timeout occurs, at which # point the currently executing cell will be skipped. If `True`, then an error # will be raised after the first timeout. This option generally does not need to # be used, but may be useful in contexts where there is the possibility of # executing notebooks with memory-consuming infinite loops. # c.ExecutePreprocessor.raise_on_iopub_timeout = False # If `graceful` (default), then the kernel is given time to clean up after # executing all cells, e.g., to execute its `atexit` hooks. If `immediate`, then # the kernel is signaled to immediately terminate. # c.ExecutePreprocessor.shutdown_kernel = 'graceful' # The time to wait (in seconds) for the kernel to start. If kernel startup takes # longer, a RuntimeError is raised. # c.ExecutePreprocessor.startup_timeout = 60 # The time to wait (in seconds) for output from executions. If a cell execution # takes longer, an exception (TimeoutError on python 3+, RuntimeError on python # 2) is raised. # # `None` or `-1` will disable the timeout. If `timeout_func` is set, it # overrides `timeout`. # c.ExecutePreprocessor.timeout = 30 # A callable which, when given the cell source as input, returns the time to # wait (in seconds) for output from cell executions. If a cell execution takes # longer, an exception (TimeoutError on python 3+, RuntimeError on python 2) is # raised. # # Returning `None` or `-1` will disable the timeout for the cell. Not setting # `timeout_func` will cause the preprocessor to default to using the `timeout` # trait for all cells. The `timeout_func` trait overrides `timeout` if it is not # `None`. # c.ExecutePreprocessor.timeout_func = None # ------------------------------------------------------------------------------ # Execute(NbGraderPreprocessor,ExecutePreprocessor) configuration # ------------------------------------------------------------------------------ # The number of times to try re-executing the notebook before throwing an error. # Generally, this shouldn't need to be set, but might be useful for CI # environments when tests are flaky. # c.Execute.execute_retries = 0 # A list of extra arguments to pass to the kernel. For python kernels, this # defaults to ``--HistoryManager.hist_file=:memory:``. For other kernels this is # just an empty list. # c.Execute.extra_arguments = [] # ------------------------------------------------------------------------------ # GetGrades(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # Preprocessor for saving grades from the database to the notebook ## # c.GetGrades.display_data_priority = ['text/html', 'application/pdf', 'text/latex', 'image/svg+xml', 'image/png', 'image/jpeg', 'text/plain'] # ------------------------------------------------------------------------------ # ClearOutputPreprocessor(Preprocessor) configuration # ------------------------------------------------------------------------------ # Removes the output from all code cells in a notebook. ## # c.ClearOutputPreprocessor.remove_metadata_fields = {'collapsed', 'scrolled'} # ------------------------------------------------------------------------------ # ClearOutput(NbGraderPreprocessor,ClearOutputPreprocessor) configuration # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ # LimitOutput(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # Preprocessor for limiting cell output # maximum number of lines of output (-1 means no limit) # c.LimitOutput.max_lines = 1000 # maximum number of traceback lines (-1 means no limit) # c.LimitOutput.max_traceback = 100 # ------------------------------------------------------------------------------ # DeduplicateIds(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor to overwrite information about grade and solution cells. # ------------------------------------------------------------------------------ # ClearHiddenTests(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # The delimiter marking the beginning of hidden tests cases # c.ClearHiddenTests.begin_test_delimeter = 'BEGIN HIDDEN TESTS' # The delimiter marking the end of hidden tests cases # c.ClearHiddenTests.end_test_delimeter = 'END HIDDEN TESTS' # Whether or not to complain if cells containing hidden test regions are not # marked as grade cells. WARNING: this will potentially cause things to break if # you are using the full nbgrader pipeline. ONLY disable this option if you are # only ever planning to use nbgrader assign. # c.ClearHiddenTests.enforce_metadata = True # ------------------------------------------------------------------------------ # OverwriteKernelspec(NbGraderPreprocessor) configuration # ------------------------------------------------------------------------------ # A preprocessor for checking the notebook kernelspec metadata. # ------------------------------------------------------------------------------ # Exchange(LoggingConfigurable) configuration # ------------------------------------------------------------------------------ # Local cache directory for nbgrader submit and nbgrader list. Defaults to # $JUPYTER_DATA_DIR/nbgrader_cache # c.Exchange.cache = '' # A key that is unique per instructor and course. This MUST be specified, either # by setting the config option, or using the --course option on the command # line. # c.Exchange.course_id = '' # Whether the path for fetching/submitting assignments should be prefixed with # the course name. If this is `False`, then the path will be something like # `./ps1`. If this is `True`, then the path will be something like # `./course123/ps1`. # c.Exchange.path_includes_course = False # The nbgrader exchange directory writable to everyone. MUST be preexisting. # c.Exchange.root = '/srv/nbgrader/exchange' # Format string for timestamps # c.Exchange.timestamp_format = '%Y-%m-%d %H:%M:%S.%f %Z' # Timezone for recording timestamps # c.Exchange.timezone = 'UTC' # ------------------------------------------------------------------------------ # ExchangeCollect(Exchange) configuration # ------------------------------------------------------------------------------ # Update existing submissions with ones that have newer timestamps. # c.ExchangeCollect.update = False # ------------------------------------------------------------------------------ # ExchangeFetch(Exchange) configuration # ------------------------------------------------------------------------------ # Whether to replace missing files on fetch # c.ExchangeFetch.replace_missing_files = False # ------------------------------------------------------------------------------ # ExchangeList(Exchange) configuration # ------------------------------------------------------------------------------ # List assignments in submission cache. # c.ExchangeList.cached = False # List inbound files rather than outbound. # c.ExchangeList.inbound = False # Remove, rather than list files. # c.ExchangeList.remove = False # ------------------------------------------------------------------------------ # ExchangeRelease(Exchange) configuration # ------------------------------------------------------------------------------ # Force overwrite existing files in the exchange. # c.ExchangeRelease.force = False # ------------------------------------------------------------------------------ # ExchangeSubmit(Exchange) configuration # ------------------------------------------------------------------------------ # Whether or not to submit the assignment if there are missing notebooks from # the released assignment notebooks. # c.ExchangeSubmit.strict = False # ------------------------------------------------------------------------------ # BaseConverter(LoggingConfigurable) configuration # ------------------------------------------------------------------------------ # Whether to overwrite existing assignments/submissions # c.BaseConverter.force = False # Permissions to set on files output by nbgrader. The default is generally read- # only (444), with the exception of nbgrader assign and nbgrader feedback, in # which case the user also has write permission. # c.BaseConverter.permissions = 0 # ------------------------------------------------------------------------------ # Assign(BaseConverter) configuration # ------------------------------------------------------------------------------ # Whether to create the assignment at runtime if it does not already exist. # c.Assign.create_assignment = False # Do not save information about the assignment into the database. # c.Assign.no_database = False # ------------------------------------------------------------------------------ # Autograde(BaseConverter) configuration # ------------------------------------------------------------------------------ # Whether to create the student at runtime if it does not already exist. # c.Autograde.create_student = False # ------------------------------------------------------------------------------ # Feedback(BaseConverter) configuration # ------------------------------------------------------------------------------ PKN)maestro/classroom_db/nb_config_example.pyPKN^t6! $maestro/classroom_db/nb_gradebook.py""" Functions to manipulate nbgrader gradebook databases. """ import os import pandas as pd class NbGradebook: """ Controls a gradebook.db database file. """ def __init__(self, path='gradebook.db'): if not os.path.exists(path): raise FileNotFoundError(path) self.path = path # Tables assignment = property(lambda self: self.table('assignment')) comment = property(lambda self: self.table('comment')) grade = property(lambda self: self.table('grade')) grade_cell = property(lambda self: self.table('grade_cell')) notebook = property(lambda self: self.table('notebook')) solution_cell = property(lambda self: self.table('solution_cell')) source_cell = property(lambda self: self.table('source_cell')) student = property(lambda self: self.table('student')) submitted_assignment = property(lambda self: self.table('submitted_assignment')) submitted_notebook = property(lambda self: self.table('submitted_notebook')) def table(self, table) -> pd.DataFrame: """ Return table as data frame. """ df = pd.read_sql(table, f'sqlite:///{self.path}') if 'id' in df.columns: df.index = df.pop('id') return df def query(self, query, **kwargs) -> pd.DataFrame: """ Return table as data frame.if __name__ == '__main__': """ return pd.read_sql_query(query, f'sqlite:///{self.path}', **kwargs) @property def max_grades(self): return self.query(''' SELECT sum(max_score) as score, nb.name as notebook FROM grade_cell as gc LEFT JOIN notebook as nb ON nb.id = gc.notebook_id GROUP BY gc.notebook_id ORDER BY score DESC ''', index_col='notebook').sort_index() def gradebook(self, normalized=False, full=True): df = self.query(''' SELECT st.id as school_id, sum(coalesce(g.auto_score, 0) + coalesce(g.manual_score, 0) + coalesce(g.extra_credit, 0)) as score, nb.name as notebook FROM grade as g LEFT JOIN submitted_notebook AS sn ON sn.id = g.notebook_id LEFT JOIN submitted_assignment AS sa ON sa.id = sn.assignment_id LEFT JOIN student AS st ON st.id = sa.student_id LEFT JOIN notebook AS nb ON nb.id = sn.notebook_id GROUP BY g.notebook_id ORDER BY notebook ASC, score DESC ''') # Remove duplicates before pivot df = df.groupby(['school_id', 'notebook']).max() df[['school_id', 'notebook']] = df.index.to_frame() grades = df.pivot('school_id', 'notebook', 'score').fillna(0) # Normalize with maximum grade possible for activity if normalized: max_grades = self.max_grades maximum = dict(zip(max_grades.index, max_grades.values)) for col in grades.columns: if col in maximum: grades[col] /= maximum[col] if full: cols = ['first_name', 'last_name', 'email'] grades[cols] = self.student[cols] grades = grades[[*cols, *grades.columns[:-3]]] return grades PKN 71 1 !maestro/classroom_db/queryable.pyfrom collections.abc import Sized, Iterable from typing import List from sidekick import op class Queryable(Sized, Iterable): """ A queryable set of elements. """ data: List['Queryable'] # # Errors # class EmptyCollection(ValueError): """ Raised on unexpected empty queries. """ class MultipleValuesFound(ValueError): """ Raised when more then the expected number of entries is found. """ def __init__(self, data=()): self.data = list(data) def __len__(self): return len(self.data) def __iter__(self): return iter(self.data) def __getitem__(self, index): return self.by_pk(index) # # Abstract methods # def by_pk(self, pk): """ Abstract method that implements how to extract elements by pk. """ raise NotImplementedError # # API # def filter(self, key=None, *, inplace=False, **kwargs): """ Filter collection by function. """ gen = iter(self.data) if key is not None: gen = filter(key, gen) # Custom filters filters = [] post_filters = [] for k, v in kwargs.items(): attr = f'filter_by_{k}' if hasattr(self, attr): post_filters.append((attr, v)) elif callable(v): filters.append((k, v)) else: filters.append((k, op.eq(v))) # Apply filters and compute result if filters: gen = (s for s in gen if all(f(getattr(s, k)) for k, f in filters)) # Apply post filters qs = self._from_generator(gen, inplace) for (attr, v) in post_filters: qs = getattr(qs, attr)(v, inplace=inplace) return qs def map(self, func, *, inplace=False): """ Map function to elements of collection. """ return self._from_generator((func(s) for s in self.data), inplace) def first(self): """ Return first element or raise ValueError if empty. """ try: return self.data[0] except: raise ValueError def last(self): """ Return last element or raise ValueError if empty. """ try: return self.data[-1] except: raise ValueError def single(self): """ Return element if collection has only one element, otherwise raise ValueError. """ if len(self) == 1: return self.data[0] elif len(self) == 0: raise self.EmptyCollection('empty collection') else: raise self.MultipleValuesFound('multiple values found') def get(self, *args, inplace=None, **kwargs): """ Like filter, but return a single element that satisfy query. Raises a ValueError if multiple elements where found. """ if inplace is not None: raise TypeError('cannot set the inplace argument for get() queries') return self.filter(*args, **kwargs).single() # # Auxiliary methods # def _from_generator(self, gen, inplace): if inplace: self.data[:] = gen return self else: return type(self)(gen) PKyNx maestro/cli/__init__.pyimport os from pathlib import Path import click import ezio from .commands import ExtractAuthorsFromZipFile, LoadFile, CheckAuthors @click.group() def cli(): pass # ============================================================================== # maestro nb # ============================================================================== @cli.group() def nb(): pass @nb.command() @click.argument('filename') @click.option( '--path', '-p', default='.', help='Base path in which nbgrader stores submissions.', ) @click.option( '--name', '-n', default=None, help='Name of activity (e.g., exam-1)', ) @click.option( '--category', '-c', default=None, help='Name of category (e.g., exams)', ) def load(filename, path, name, category): """ Extract user names from values. """ from .nb import load_zip if category is None: category = os.path.splitext(os.path.basename(filename))[0] if name is None: name = os.path.splitext(os.path.basename(filename))[0] load_zip(Path(filename), Path(path), name, category) @nb.command() @click.option( '--path', '-p', default='./gradebook.db', help='Base path to the gradebook.db database.', ) @click.option( '--output', '-o', default=None, help='Output file.', ) @click.option( '--simple/--full', default=False, help='Display only minimal information.', ) @click.option( '--normalize', '-n', type=float, default=None, help='Normalize all grades relative to the maximum grade.', ) @click.option( '--sort', '-s', default='id', help='Sort according to column', ) def export(path, output, simple, normalize, sort): """ Export grading data as comma separated values. """ from ..classroom_db.nb_gradebook import NbGradebook gb = NbGradebook(path) df = gb.gradebook(full=not simple, normalized=normalize is not None) if normalize: n = 2 if not simple else 0 df[df.columns[n:]] *= normalize if sort == 'id': df = df.sort_index() elif sort in df.columns: df = df.sort_values(sort) else: msg = f'ERROR! Invalid column to sort: {sort}' cols = ', '.join(df.columns) ezio.print(msg, format=True) ezio.print(f'Columns must be one of {cols}, or id') raise SystemExit() # Write if output: df.to_csv(output, float_format='%.2f') else: print(df.to_csv(float_format='%.2f')) # ============================================================================== # Main # ============================================================================== def main(): return cli() if __name__ == '__main__': main() PK=NOzATTmaestro/cli/commands.pyimport json import os import zipfile from pathlib import Path import click from maestro.cli.config import TomlConfigFile from .fuzzy_match import FuzzyMatcher from .utils import dict_merge class NbCommand: """ Abstract class for many notebook management commands. """ @classmethod def exec(cls, *args, **kwargs): """ Execute task while creates it. """ task = cls(*args, **kwargs) task.run() return task def __init__(self, path, config_path): self.path = path self.config_path = config_path self.invalid_files = {} self.config = TomlConfigFile(Path(config_path) / 'config.toml') try: self.zipfile = zipfile.ZipFile(self.path) except zipfile.BadZipFile as ex: msg = click.style(str(ex), bold=True, fg='red') click.echo(msg, err=True, color=True) exit() # # Public methods # def run(self): """ Abstract implementation. """ raise NotImplementedError def load_files_info(self): """ Return the authors list. """ authors = {} paths = sorted(self.zipfile.filelist, key=lambda x: x.date_time) for path in paths: with self.zipfile.open(path) as fd: try: data = json.load(fd) names = self.extract_author_name(data) except json.JSONDecodeError: fd.seek(0) self.invalid_files[fd.name] = fd.read() else: authors.update({name: path for name in names}) return authors def update_config(self, data): """ Update config file with the given data. """ self.config.data = dict_merge(self.config.data, data) self.config.save() class ExtractAuthorsFromZipFile(NbCommand): """ Extract authors from zip file of submissions. """ def run(self): """ Load files asking interactively. """ authors = sorted(set(self.load_files_info())) # Show list of authors click.echo('List of users:') for n, author in enumerate(authors, start=1): click.echo(f'{n}) {author}') # Clean list and return remove = self._ask_remove() return [x[1] for x in enumerate(authors) if x not in remove] def _ask_remove(self): remove = click.prompt('Remove (comma separated)', default='', show_default=False) numbers = remove.strip().split(',') try: return [int(x) - 1 for x in numbers if x] except ValueError: click.echo('Invalid input!') return self._ask_remove() class CheckAuthors(NbCommand): """ Match authors to registered authors """ def __init__(self, path, config_path, name=None, category=None): super().__init__(path, config_path) self.pending = {} self.files_map = {} self.name = name self.category = category def run(self): """ Match all given names with the corresponding real values. """ config = self.config # Start fuzzy matcher files = self.load_files_info() real = config.get('students.ids', []) matcher = FuzzyMatcher(files.keys(), real) # Remove certain matches author_map = matcher.remove_exact(config.get_all_student_aliases()) matcher.fill_distances() matcher.set_distance_threshold(0.90) # Match each missing author with the given real name while matcher.shape[0] != 0 and False: given_, real_ = matcher.closest_pair() click.echo(f'\nBest match for {given_}') matches = self.ask_matches(given_, matcher.best_matches(given_, 5)) if matches: for match in matches: matcher.remove_pair(given_, match) config.add_student_aliases(match, [given_]) author_map[given_] = match else: matcher.remove_given(given_) # Save files read_zip = lambda x: self.zipfile.open(x).read() for k, f in files.items(): if k in author_map: data = read_zip(f.filename) for name in author_map[k]: path = Path(f'submitted/{name}/{self.category}/{self.name}.ipynb') path.parent.mkdir(parents=True, exist_ok=True) if not os.path.exists(path): with open(path, 'wb') as fd: fd.write(data) def ask_matches(self, value, options): """ Associate match for value from list of options. """ # Fast track dominant matches if len(options) == 1 or options[1][1] - options[0][1] > 0.5: match = options[0][0] if click.confirm(f'Confirm as {match}?', default=True): return [match] # Print menu for i, (m, pc) in enumerate(options, start=1): pc = int(100 * (1 - pc)) click.echo(f' {i}) {m} ({pc}%)') click.echo(f'\n {i + 1}) Add new') click.echo(f' {i + 2}) Ignore') # Process value while True: try: opts = self.ask_options('Choose option', len(options), -1) except ValueError: click.echo('Invalid option!') continue if opts == 'add-new': self.config.add_student(value) return [value] elif opts == 'ignore': return [] else: return opts def ask_options(self, msg, n_options, delta=0): """ Ask for one or more options in a list of n elements. """ opt_max = n_options new_option = n_options + 1 ignore_option = n_options + 2 res = map(int, click.prompt(msg, type=str).split(',')) if res == [new_option]: return 'add-new' elif res == [ignore_option]: return 'ignore' elif all(1 <= n <= opt_max for n in res): return [n + delta for n in res] class LoadFile(NbCommand): def __init__(self, path, config_path, name=None): super().__init__(path, config_path) self.pending = {} # Save name if name is None: name = os.path.splitext(os.path.basename(path))[0] self.name = name # Authors map self.files_info = self.load_files_info() self.author_map = {} def run(self): self.check_authors(self.files_info.keys(), self.config.data['students']['ids']) # for author, name in authors.items(): # print('author', author, 'name', name) if self.invalid_files: self.handle_invalid_files() def handle_invalid_files(self): n = len(self.invalid_files) error_path = os.path.join(self.config_path, 'error') click.echo(f'{n} invalid files were saved at {error_path}/') try: os.mkdir(error_path) except FileExistsError: pass for name, data in self.invalid_files.items(): name = os.path.basename(name) with open(os.path.join(self.config_path, 'error', name), 'wb') as fd: fd.write(data) PKypNO maestro/cli/config.pyfrom collections.abc import MutableMapping import toml from sidekick import lazy class TomlConfigFile(MutableMapping): """ Represents a TOML config file. """ @lazy def data(self): try: return toml.load(open(self.path)) except FileNotFoundError: return {} def __init__(self, path): self.path = path def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] def __getitem__(self, key): return self.data[key] def __len__(self): return len(self.data) def __iter__(self): return iter(self.data) def save(self): """ Forces saving to disk. """ toml.dump(self.data, open(self.path, 'w')) def get(self, path, default=None): """ Like dict's get, but understand the dot notation to designate sub-elements. """ if '.' in path: data = self.data *paths, last = path.split('.') for fragment in paths: data = data.get(fragment, {}) return data.get(last, default) return super().get(path, default) def add_student(self, id_, save=True): """ Register student id. """ ids = self.data.setdefault('students', {}).get('ids', []) if id_ not in ids: ids.append(id_) ids.sort() if save: self.save() def get_student(self, id_): """ Return information about student with given id. """ return {'id': id_} def add_student_aliases(self, id_, aliases, save=True): """ Register list of aliases for the given student id. """ alias_db = self.data \ .setdefault('students', {}) \ .setdefault('aliases', {}) for alias in aliases: lst = alias_db.setdefault(alias, []) if id_ not in lst: lst.append(id_) if save: self.save() def get_student_aliases(self, id_): """ Return list of aliases for student. """ return self.get(f'students.aliases.{id}', []) def get_all_student_aliases(self): """ Return a mapping from alias to real ids for all students. """ aliases = dict(self.get('students.aliases', ())) aliases.update({id_: [id_] for id_ in self.get('students.ids')}) return aliases PKtNlm maestro/cli/fuzzy_match.pyimport sidekick as sk class FuzzyMatcher: """ Fuzzy match 2 lists of strings. """ given: list real: list shape = property(lambda self: (len(self.given), len(self.real))) def __init__(self, given, real): self.given = list(given) self.real = list(real) self.distances = {} def remove_exact(self, aliases=None): """ Remove all exact matches from matcher. """ exact = {} for x in list(self.given): if x in self.real: self.remove_pair(x, x) exact[x] = [x] # Check if any alias is present if aliases: for x in list(self.given): if x in aliases: ys = aliases[x] for y in ys: try: self.remove_pair(x, y) except ValueError: pass exact[x] = ys return exact def remove_pair(self, a, b): """ Remove pair if present in matcher. """ if b in self.real: self.given.remove(a) self.real.remove(b) self.distances.pop((a, b), None) else: raise ValueError self.distances = { (a_, b_): d for ((a_, b_), d) in self.distances.items() if a != a_ and b_ != b } def remove_given(self, a): self.given.remove(a) self.distances = {(a_, b): d for ((a_, b), d) in self.distances.items() if a != a_} def set_distance_threshold(self, value): """ Remove all pairs whose distance is larger than or equal the given threshold. """ for pair, dist in list(self.distances.items()): if dist >= value: del self.distances[pair] def fill_distances(self): """ Fill distance matrix. """ matrix = self.distances for x in self.given: for y in self.real: matrix[x, y] = pair_distance(x, y) return matrix def _fill_distances_non_empty(self): if not self.distances: self.fill_distances() def closest_pair(self): """ Return the pair of values with the smallest distance. """ self._fill_distances_non_empty() return min(self.distances, key=self.distances.get) def best_matches(self, given, n): """ Provide the n best matches for the given value. """ self._fill_distances_non_empty() return sorted( ((b, d) for ((a, b), d) in self.distances.items() if a == given), key=lambda x: x[1] )[:n] def set_distance(a, b, sep='_'): sa = set(a.split(sep)) sb = set(b.split(sep)) return len(sa.symmetric_difference(sb)) / len(sa.union(sb)) def pair_distance(a, b, sep='_'): a = a.split(sep) b = b.split(sep) sa = set(bigrams(a)).union(a) sb = set(bigrams(b)).union(b) return len(sa.symmetric_difference(sb)) / len(sa.union(sb)) def bigrams(lst): return list(sk.window(2, lst)) PKN@gmaestro/cli/nb.pyimport json import os import re import zipfile from hashlib import md5 from pathlib import Path from typing import Optional import ezio import sidekick as sk from ezio import fs from ezio import print as echo, input as ask from maestro.classroom_db import Students from ..classroom_db import Classroom, Student class File(sk.Union): """ Represents a loaded file. """ Ok: sk.Case(name=str, data=dict) Err: sk.Case(name=str, data=bytes, msg=str) def load_zip(zip_path: Path, config_path: Path, name: str, category: str): """ Load files from given zip file at config path and save in the corresponding locations in the submitted folder. """ echo = ezio.print ask = ezio.input classroom = Classroom.load(Path(config_path) / 'config.toml') students: Students = classroom.students extract_names = AuthorExtractor().extract bad_files = [] existing_files = [] for file in load_zip_files(zip_path): msg = f'Loading data from file {file.name}' echo(msg, format=True, end='') if not file.is_ok: echo('.') bad_files.append(file.name) handle_bad_file(file, config_path) continue found_students = extract_names(file.data) echo(f' ({len(found_students)} students found).') for student in found_students: found = students.best_match(student) # Happy case if found: student = found # Student was not found in database: ask for inclusion elif student.school_id: msg = f' {student.display} not found.\n Add to database?' if ask(msg, type=bool, default=True, format=True): students.add(student) classroom.save() echo() else: echo('User skipped.') continue # Student was not found and has no school id. # We need to ask a valid school id to the user. else: student = ask_for_valid_school_id(student.name, classroom) if not student: continue # Save student data assert student.school_id, student save_path = os.path.join(config_path, 'submitted', student.school_id, category, name + '.ipynb') if not os.path.exists(save_path): fs.write(save_path, json.dumps(file.data), make_parent=True) else: existing_files.append(save_path) # Global messages if existing_files: msg = '\nWARNING! Skipping existing files:' echo(msg, format=True) for file in existing_files: echo(f' * {file}', format=True) if bad_files: msg = '\nERROR! The following files are invalid' echo(msg, format=True) for file in bad_files: echo(f' * {file}', format=True) def load_zip_files(zip_path): """ Iterate over all File's in the given zip file. """ zd = zipfile.ZipFile(zip_path) paths = sorted(zd.filelist, key=lambda x: x.date_time) for path in paths: with zd.open(path, 'r') as fd: try: data = json.load(fd) yield File.Ok(fd.name, data) except (json.JSONDecodeError, UnicodeDecodeError) as exc: fd.seek(0) yield File.Err(fd.name, fd.read(), str(exc)) def handle_bad_file(file: File, config_path: Path): """ Saves invalid files in the error folder and prints a message to user. """ msg = 'ERROR! Invalid file type/content.' echo(msg, format=True) # Save file fname, ext = os.path.splitext(os.path.basename(file.name)) md5_hash = md5(file.data).hexdigest() path = config_path / 'error' / f'{fname.lower()}-{md5_hash}{ext}' ezio.fs.write(path, file.data, '-p') echo(f'File saved under {path}') def ask_for_valid_school_id(name: str, classroom: Classroom) -> Optional[Student]: """ Construct new valid student for classroom asking user for a valid school id. """ msg = f'{name} not found.' echo(msg, format=True) while True: school_id = ask('School id: ', default=None) if school_id is None: if ask('Skip student? ', type=bool): echo() return None else: continue try: student = Student(name, school_id) classroom.students.add(student) classroom.save() return student except ValueError: echo(f'School id exists for {student.display}') if ask('Reuse? ', type=bool): student = classroom.students.get(school_id=school_id) student.aliases.append(student.name) classroom.save() echo() return None class AuthorExtractor: """ Extract author name from ipynb file. """ id_re = re.compile(r'\d+/?\d+') def __call__(self, *args, **kwargs): return self.extract(*args, **kwargs) def extract(self, data): """ Return a list of authors from ipynb data. """ # Read authors from the cell with NAME and COLLABORATORS variables authors = [] for cell in data['cells']: if cell['cell_type'] != 'code': continue src = cell['source'] if src and src[0].startswith('NAME'): authors = [*self._extract_name(src, 'NAME'), *self._extract_name(src, 'COLLABORATORS')] break return [x for x in authors if x.name] def _extract_name(self, src, variable='COLLABORATORS'): for line in src: if line.startswith(variable): _, _, name = line.partition('=') return [self._with_student_id(name.strip(' "\''))] return [] def _with_student_id(self, name): m = self.id_re.search(name) if m: i, j = m.span() id_ = name[i:j].replace('/', '') name = name[:i] + name[j:] else: id_ = None for symb in '[]{}(),.;:-"\'\t\n\r\f/\\?@$%!': name = name.replace(symb, '') name = ' '.join(name.split()) if name.isupper() or name.islower(): name = name.title() student = Student(name, school_id=id_) student.aliases.append(student.normalized_name) return student PKsNmӴmaestro/cli/utils.pydef dict_merge(a, b): a_ = dict(a) _dict_merge(a_, b) return a_ def _dict_merge(a, b): # Merges b into a *inplace* for kb, vb in b.items(): if isinstance(vb, dict): try: va = a[kb] except KeyError: a[kb] = vb.copy() else: _dict_merge(va, vb) else: a[kb] = vbPKkOCOmaestro/course/__init__.pyPKCOܗ55$maestro/course/compilers/__init__.py""" Functions that might be useful to test code and assignments in a Compilers course. Some functions might also be useful elsewhere (e.g.. regex testing). """ from .regex import check_regex, RegexTester from .regex_crosswords import ( regex_crossword_table, check_regex_crossword, CROSSWORD_EXAMPLES, ) PK`DO J!maestro/course/compilers/regex.pyimport io import re from typing import Pattern, Tuple, Union from sidekick import lazy TupleSt = Tuple[str, ...] def check_regex(regex, spec, accept_empty=False, max_size=float('inf'), silent=False): """ Check if regular expression accepts all "ok" strings and and refuses "negative" strings. Raises an AssertionError if regular expression is invalid. If negative is not given and ok is a string, it interpret as a multi-line string Args: regex (str): A string representing a regular expression. spec (str): A multiline string split into 2 sections with POSITIVE and NEGATIVE examples. Each example is in its own line. accept_empty (bool): If True, accepts empty strings. max_size (int): Expected size of string. silent (bool): If true, omit messages on success. """ kwargs = {'accept_empty': accept_empty, 'max_size': max_size} return RegexTester.from_spec(regex, spec, **kwargs).run(silent=silent) class RegexTester: """ Test a group of strings against a regular expression. """ _congratulations_message = "Congratulations! Your regex is correct :)" wrongly_accepted: TupleSt = lazy(lambda _: tuple(filter(_.rejects, _.positive))) wrongly_rejected: TupleSt = lazy(lambda _: tuple(filter(_.accepts, _.negative))) size: int = lazy(lambda _: len(_.pattern)) regex: Pattern pattern: str positive: TupleSt negative: TupleSt size_multiplier: float max_size: Union[float, int] accept_empty: bool @classmethod def from_spec(cls, regex: str, spec: str, **kwargs): """ Construct regex tester from spec. """ kwargs.update(parse_spec(spec)) return cls(regex, **kwargs) @lazy def grade(self) -> float: grade = 1.0 - bool(self.wrongly_accepted or self.wrongly_rejected) if not self.accept_empty and self.accepts(''): return 0.0 if self.size > self.max_size: return self.size_multiplier * grade return grade def __init__(self, regex, positive=(), negative=(), max_size=float('inf'), size_multiplier=0.75, accept_empty=False): if isinstance(regex, str): self.pattern = regex self.regex = re.compile(self.pattern) else: self.regex = regex self.pattern = self.regex.pattern self.positive = tuple(positive) self.negative = tuple(negative) self.size_multiplier = size_multiplier self.max_size = max_size self.accept_empty = accept_empty def accepts(self, st): """Check regular expression accepts string.""" return self.regex.fullmatch(st) is not None def rejects(self, st): """Check if regular expression rejects string.""" return not self.accepts(st) def run(self, silent=False, exception=AssertionError): """Check if regular expression passes all tests.""" if self.grade == 1.0: if not silent: n = len(self.pattern) print(f'Congratulations! Valid regular expression with {n} characters.') return raise exception(self.message()) def check_size(self, multiplier=None, silent=False, exception=AssertionError): """ Checks only if size is correct. """ if len(self.pattern) > self.max_size: raise exception(self.size_message(multiplier)) elif not silent: print(self._congratulations_message) def message(self, skip_size=False): """ Return an error or congratulatory message. """ if self.grade == 1.0: return self._congratulations_message fd = io.StringIO() fd.write(f'Invalid regular expression: /{self.pattern}/\n\n') if not self.accept_empty and self.accepts(''): fd.write('Regular expression *must not* accept empty strings\n\n') self._warn_examples(fd, must='accept', at=self.wrongly_accepted) self._warn_examples(fd, must='reject', at=self.wrongly_rejected) skip_size = skip_size or self.max_size > len(self.pattern) if not skip_size: fd.write(self.size_message(congratulate=False)) grade = int(100 * self.grade) fd.write(f"Your final grade is {grade}%.\n") return fd.getvalue() def size_message(self, congratulate=True): """ Check if size is under budget and return an error or congratulatory message. """ fd = io.StringIO() n = len(self.pattern) - self.max_size if n > 0: fd.write(f'Regex must be {n} character smaller to receive full grade.\n') fd.write('\n') elif congratulate: fd.write(self._congratulations_message) return fd.getvalue() def _warn_examples(self, fd, must, at): if at: fd.write(f'Regular expression must *{must}* the examples\n') for st in at: fd.write(f' - {st!r}\n') fd.write('\n') else: fd.write(f'Regular expresion correctly *{must}s* all examples :)\n\n') def parse_spec(spec: str) -> dict: """ Return a map of {"positive": [], "negative": []} Examples: >>> parse_spec(''' ... POSITIVE: ... ab ... abab ... ... NEGATIVE: ... ba ... baba ... ''') {'positive': ['ab', 'abab'], 'negative': ['ba', 'baba']} """ _, sep1, spec = spec.strip().partition('POSITIVE:\n') positive, sep2, negative = spec.strip().partition('NEGATIVE:\n') if not sep1 or not sep2: raise ValueError('Spec does not contain a POSITIVE: and NEGATIVE: sections.') return { 'positive': positive.strip().splitlines(), 'negative': negative.strip().splitlines(), } PK(CO+,Q ,maestro/course/compilers/regex_crosswords.pyfrom collections import namedtuple import re from hyperpython import table, tr, td, th, div Ex = namedtuple('Example', ['rows', 'cols']) def check_regex_crossword(answer, rows, cols, silent=False): r""" A regex crossword is defined by a list of rows and a list of columns containing regular expression strings. A valid answer is a rectangular table of shape [n, m] (where `n = len(rows)` and `m = len(cols)`) in which every string formed by concatenating elements in a row must match the corresponding column regex and likewise for rows. Example: >>> ans = [ ... ["1", "2"], ... ["3", "4"] ... ] >>> check_regex_crossword(ans, cols=[r"12|34", r".\d"], rows=[r"13|24", r"[123][a4]"]) Congratulations! """ if isinstance(answer, str): answer = answer.strip('\n').splitlines() for i, expr in enumerate(rows): row = ''.join(answer[i]) if not re.fullmatch(expr, row): raise AssertionError(f'Does not match row {i + 1} /{expr}/: {row!r}') for i, expr in enumerate(cols): col = ''.join(line[i] for line in answer) if not re.fullmatch(expr, col): raise AssertionError(f'Does not match column {i + 1} /{expr}/: {col!r}') if not silent: print('Congratulations!') def regex_crossword_table(rows, cols): """ Render a regular expression crossword table from its defining rows and columns. """ head_style = ( "transform: rotate(-90deg) translate(0, 1rem); " "transform-origin: 0% 0%; " "height: 1.5rem; " "width: 12rem; " "text-align: left; " "position: absolute; " "font-size: 1.5rem" ) td_style = ( "width: 4rem; " "height: 4rem; " "border: 1px solid black;" ) table_style = ( "transform: rotate(30deg); " "font-family: monospace; " "margin: auto" ) return div(style="margin: 3rem auto; padding-right: 5rem;")[ table(style=table_style)[[ # Head tr(style="height: 12rem; background: none")[[ th(), *[th(div(r, style=head_style), style="vertical-align: bottom") for r in cols] ]], # Body rows *(tr([ th(r, style="font-size: 1.5rem"), *(td(" ", style=td_style) for _ in cols) ]) for r in rows) ]] ] # # Library of examples (safe to use since it does not contain answers). # CROSSWORD_EXAMPLES = { # Initial examples "abba": Ex( rows=["ab", "cd"], cols=["ac", "db"], ), "abc?": Ex( rows=["ab?c?", "c?d?e?"], cols=["a+", "ca*d*"], ), # Test, 10/2019 "john": Ex( rows=["[AWE]+", "[ALP]+K", "(PR|ER|EP)"], cols=["[BQW](PR|LE)", "[RANK]+"], ), "yr": Ex( rows=[r"18|19|20", r"[6789]\d"], cols=[r"\d[2480]", r"56|94|73"], ), } PKLOCOƂ!maestro/grader/function_grader.pyimport argparse def repr_fcall(fname, args, kwargs): """Nice string representation for function call""" data = ', '.join(map(repr, args)) data += ', '.join('%s=%r' % item for item in kwargs.items()) return '%s(%s)' % (fname, data) def function_grader(mod, func_name, examples, print=print): """ Grades a function using a list of examples. """ try: func = getattr(mod, func_name) except AttributeError: msg = 'Error: module does not have a %r function [grade = 0%%]' raise SystemExit(msg % func_name) n_ok = 0 n_items = len(examples) print('Grading %s examples...' % n_items) for args, kwargs, expected in examples: correct = False try: result = func(*args, **kwargs) correct = result == expected except Exception as ex: error_name = ex.__class__.__name__ fcall = repr_fcall(func_name, args, kwargs) print(' %s at %s: %s' % (error_name, fcall, ex)) else: if correct: n_ok += 1 else: fcall = repr_fcall(func_name, args, kwargs) msg = ' wrong: %s -> %r, expected %r.' print(msg % (fcall, result, expected)) # Final summary if n_ok == n_items: print('Congratulations! All %s tests passed!' % n_ok) else: pc = 100 * n_ok / n_items print('Success rate: %.0f%% (%d/%d)' % (pc, n_ok, n_items)) PKJK}e**maestro/regex/__init__.pyfrom .random_strings import random_string PKOTKfBL<<maestro/regex/json.py""" JSON (http://json.org/) is a very popular data serialization format designed to be a data-driven subset of Javascript. The website "json.org" has nice diagrams explaining the details of the format. In this exercise we are going to implement the regular expressions necessary to create a JSON lexer. """ class Open_brace: """ From the spec: "An object is an unordered set of name/value pairs. An object begins with { (left brace) and ends with } (right brace). Each name is followed by : (colon) and the name/value pairs are separated by , (comma)." Let us start easy: write a regular expression that matches only the opening brace. """ regex = r'\{' class Control_chars: """ Now we want an expression that matches any other control character as described in the full spec. Read the description of the format on the website in order to identify all control characters. """ regex = r'[:[{},\]]' max_length = len(regex) class Keywords: """ JSON also have a few keywords representing booleans and nulls. Match them! """ regex = r'true|false|null' max_length = len(regex) accept = 'true false null' reject = 'True False Null None' class Integer: """ Now things start to get complicated... Lets crack the number spec: you must create a regex that matches anything before the dot. Ignore scientific notation for now. """ regex = r'\-?(0|[1-9][0-9]*)' max_length = 2 * len(regex) accept = '0 1 2 42 -1' reject = '01 3.14' class Simple_float: """ Add a suffix to the previous regex that matches a number with an optional decimal part. """ regex = r'\-?(0|[1-9][0-9]*)(\.[0-9]+)?' max_length = 2 * len(regex) accept = '42 3.14' reject = '01 3a14 3. .42' class Number: """ Finally, include the optional exponent for scientific notation. If you haven't noticed yet, it is possible to construct regular expressions by combining strings of simpler regular expressions. You can declare it like: regex = Simple_float.regex + r'(scientific notation regex)' """ regex = Simple_float.regex + r'([eE][+-]?[0-9]+)?' max_length = 2 * len(regex) accept = '42 3.14' reject = '01 3a14 3. .42' class Simple_string: """ We start by matching a very simple string that do not contain any special characters. It must be enclosed by double quotes and can include any character except \ (backslash) or " (double quote) """ regex = r'"[^\\"]*"' accept = ['"hello"', '""', '"hello world!"'] reject = ['"\n"', '"bad"string"'] class String_with_escaped_double_quote: """ Add support for escaping a inner double quote as in this ugly example: "hello \"World\"!" """ regex = r'"([^\\"]*(\\")?)+"' accept = ['"hello"', '""', '"hello world!"'] reject = ['"\n"', '"bad"string"'] class Full_String: """ Add support for special escaping sequences specified on the spec. """ regex = r'"([^\\"\n]*(\\([bfnrt"/\]|u[0-9a-fA-F]{4}))?)+"' accept = ['"hello"', '""', '"hello world!"'] reject = ['"\n"', '"bad"string"'] # # HJSON # class Line_comments: """ HJSON extends JSON by (among other things) allowing comments. It accepts either C-style or hash-style line comments. """ regex = r'(//|#)[^\n]*' accept = ['// a C-like comment', '# a Python like comment'] reject = ['# trailing newlines are not part of the comment\n'] class Block_comments: """ HJSON also accepts C-style block comments. Match them! """ regex = r'/\*([^*]*[^/])*\*/' accept = ['/* block comment */'] reject = ['// a C-like comment', '# a Python like comment'] # # TOML # class Multi_line_bare_strings: """ Just like Python, TOML accept multi-line string literals. Those literals can contain new lines and are enclosed in 3 single quotes. """ regex = r"'''([^']*'?)*'''" class Bare_strings: """ TOML has the concept of bare strings. They work similarly to regular strings, but they do not support escaping. Backslashes are not treated as control characters, but rather as literal backslashes. Bare strings are enclosed in single quotes and cannot have newlines or the line feed character. """ regex = r"'[^'\n\r]'" class Strings: """ Let us start with strings. In TOML, string literals follow basically the same rules as JSON, but they also allow for unicode code points with 8 digits to be escaped with an uppercase U. """ regex = r'"([^\\"\n\r]*(\\([bfnrt"/\]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}))?)+"' accept = [r'"unicode: \U02468AD"'] reject = [r'"broken unicode: \U02468"'] class Multi_line_strings: """ Standard multi-line strings are enclosed in triple quotes and accept escaping just like regular strings. The only difference between regular strings and multi line strings is the the later can contain line breaks and are enclosed by 3 double quotes, instead of one. Create a regex that match a multi line string. """ regex = r'"([^\\"]*(\\([bfnrt"/\]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}))?)+"' accept = [r'"unicode: \U02468AD"'] reject = [r'"broken unicode: \U02468"'] class Integer: """ In TOML, integers can be prefixed with a sign and can contain underscores in order to group digits in a human-friendly way. """ regex = r'[-+]?(0|[1-9](_?[0-9]+)*' accept = '0 1 2 42 -42 +42 1_234 1_23_45_678' reject = '01 _ _1 2_ ++1 A42 1__2' class Binary: """ Binary literals start with 0b and can have one or binary digits (0 or 1). Underscores are allowed. """ regex = r'[-+]?0b[01](_[01]+)*' accept = '0b11 +0b10 -0b01 0b1010_1101' class Octal: """ Octal follow similar rules of binary literals, but uses the 0o prefix and (obviously) accept any octal digits. """ regex = r'[-+]?0o[0-8](_[0-8]+)*' accept = '0o42 +0b12_34' class Hexadecimal: """ Hexadecimal literals start with 0x and can have one or more hexadecimal digits. Hexadecimal digits can be lowercase or uppercase. """ regex = r'[-+]?0x[0-9a-fA-F](_[0-9a-fA-F]+)*' accept = '0x42 0xff 0xFF 0x123_abc' class Decimal: """ Floats add a decimal part and a scientific notation to integers. Implement a regex that adds a required decimal part that follows similar rules as integers. """ regex = r'[-+]?(0|[1-9](_?[0-9]+)*\.([0-9](_?[0-9]+)*' class Floats: """ Now implement the optional exponent part. """ regex = Decimal.regex + r'([Ee][-+]?[1-9](_?[0-9]+)*)?' class Float_literals: """ TOML understands a few floating point literals for special floating point values. It understands inf and nan in signed and unsigned forms. """ regex = r'[+-]?(inf|nan)' class Date: """ TOML accept RFC 3339 datetime values. Unfortunately the whole spec is too complicated to be implemented with regular expressions. Let us start with a pattern to match only simple dates in the YYYY-MM-DD format. Although it is feasible, it is not considered to be a good practice to validate the days and values using the regex itself. """ regex = r'[0-9]{4}-[0-9]{2}-[0-9]{2}' class Time: """ Time is given in the hh:mm:ss or hh:mm:ss.ssssss formats. """ regex = r'[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{6})' class Datetime: """ A full datetime format can be given by joining the date and time parts either with an uppercase T or an space. """ regex = Date.regex + r'[T ]' + Time.regex class Full_datetime: """ The full format involves a datetime + a timezone specification. The timezone can be given as an offset in the format (+|-)hh:mm or as the literal Z letter to specify UTC. """ regex = Datetime.regex + r'(Z|[0-9]{2}:[0-9]{2})' # JSON parsers import ox lexer = ox.make_lexer([ ('LBRACE', r'\{'), ('RBRACE', r'\}'), ('LBRACKET', r'\['), ('RBRACKET', r'\]'), ('COMMA', r','), ('COLON', r':'), ('STRING', Full_String.regex), ('NUMBER', Number.regex), ('BOOL', r'true|false'), ('NULL', r'null'), ]) def to_string(x): return x[1:-1] # cheat! parser = ox.make_parser([ # Simple values ('value : object', lambda x: x), ('value : array', lambda x: x), ('value : NUMBER', lambda x: float(x)), ('value : BOOL', lambda x: x == 'true'), ('value : NULL', lambda x: None), ('value : STRING', lambda x: to_string(x)), ('object : LBRACE RBRACE', lambda _, __: {}), ('object : LBRACE members RBRACE', lambda _, pairs, __: dict(pairs)), ('members : pair', lambda x: x), ('members : pair COMMA members', lambda x, _, xs: [x] + xs), ('pair : STRING COLON value', lambda k, _, v: (k, v)), ('array : LBRACKET RBRACKET', lambda _, __: {}), ('array : LBRACKET elements RBRACKET', lambda _, xs, __: xs), ('elements : value', lambda x: [x]), ('elements : value COMMA elements', lambda x, _, xs: [x] + xs), ], tokens=[ 'LBRACE', 'RBRACE', 'LBRACKET', 'RBRACKET', 'COMMA', 'COLON', 'STRING', 'NUMBER', 'BOOL', 'NULL']) """ Regex 101 """ class Simple_match: """ In in its simplest form, a regex simply compares itself with another string. For instance, the regex r'hello' simply matches the string 'hello'. This is not very useful, but nevertheless, write a regex that matches the string 'hello regex!' """ regex = r'' class Group_of_characters: """ What distinguishes regexes from simple string match is that some characters have special meanings and can be used to match more flexible patterns. By enclosing a group of characters inside brackets [abcde...] we are declaring that we accept any of them in the given position. Create a regular expression that matches either the string 'Hello!' or the string 'hello!'. """ regex = r'[hH]ello!' class Ranges: """ The [] grouping is very powerful. We can declare a group of characters or even a range of values. If characters appear sequentially (e.g., a, b, c, ...) we can declare them as a ranges [a-cz-f]. Create a grouping that matches any digit of an hexadecimal number. """ regex = r'[0-9a-fA-F]' class Be_careful_with_spaces: """ Spaces inside a regex string are usually treated as literal space characters. There are special flags that can disable this behavior by simply ignoring spaces inside a regex string. Create a regex that matches a vowel followed by a single space and then a digit. """ regex = r'[aeiou] [0-9]' class Negated_matches: """ Sometimes is useful to accept any character except the characters in a group. We can negate a group of characters by starting the grouping with a caret such as in [^abc]. Create a regex that matches anything but numbers. """ regex = r'[^0-9]' class Repetitions: """ Now something very useful: we can declare repetitions of a pattern. Regexes offer 3 basic ways to declare repetitions (we will look into more advanced patterns later). x* -- zero or more repetitions of x x+ -- one or more repetitions of x x? -- one or zero repetitions of x (that is the same as saying that x is optional) Of course we can exchange x by any other character or even by groups of characters (such as in [abc]*) Create a regex that matches an integer number: it can have an optional plus or minus sign followed by a sequence of digits. """ regex = r'[+-]?[0-9]+' max_length = len(regex) class Advanced_repetitions: """ We can control the number of repetitions more precisely using curly brackets: x{n} -- n repetitions x{n,m} -- between n and m repetitions x{n,} -- at least n repetitions x{,m} -- at most m repetitions Create a regular expression that matches an ISO date of form YYYY-MM-DD """ regex = r'[0-9]{4}-[0-9]{2}-[0-9]{2}' max_length = len(regex) class Alternatives: """ The [] notation allow us to choose many different characters that might appear on a given location. Sometimes we want to declare possible alternative substrings. That is what the pipe operator | is for: r'pattern1|pattern2' will match either pattern1 or pattern2, independently of the number of characters. Create a regex that matches either 'True' or 'False' """ regex = r'True|False' max_length = len(regex) class Grouping: """ Sometimes we need to explicitly declare substrings in repetitions or as alternatives. Enclose the sub-pattern in parenthesis to define a group that is handled as a unity. Create a regex that matches either 'Hello world!' or 'Hello regex!' """ regex = r'Hello (world|regex)!' max_length = len(regex) class Grouping_hard: """ Now modify your regex to match strings of the form 'Hello !' in which is composed by one or more parts composed by a name that starts with an uppercase letter followed by one or more lowercase letters. """ regex = r'Hello ([A-Z][a-z]+( [A-Z][a-z]+)*)!' class Escaping: r""" What happens if you need to match an special character such as (, [ or +? Regexes allow us to escape them by preceeding the character with a backslash such as \+ or \\. Create a regex that matches strings of the form " ()" """ regex = r'[A-Z][a-z]+ \([0-9]+\)' class Escaping_inside_a_group: r""" All characters except ] and \ loose their special behavior inside bracket matches. The dash (-) and caret (^) can be escaped in order to loose their special meaning inside a bracket group. Create an expression that matches expressions of the form "" such as "1+1" """ regex = r'[0-9]+[-+*/^][0-9]+' # # Python basic # # Variables and basic operations # Executing functions # Basic string # Print and input """ Interaction =========== author: Fábio Macêdo Mendes email: fabiomacedomendes@gmail.com Python has two main functions for interacting with the user. The input(msg) asks the user for some input and return it as a string. The print(obj) function, shows the content of the input on the screen. Create a program that asks for the user name and save it on a variable called *name*. Afterwards, it should print the message "Hello !", replacing with the given name. Example: Name: Hello John Lennon! (User input is between .) """ from maestro import fixtures name = fixtures.name() def test_module(code): result = run_code(code, inputs=['John Lennon']) assert result == (..., 'John Lennon', 'Hello John Lennon!') def test_synthetic_examples(code, name): result = run_code(code, inputs=[name]) assert result == (..., name, 'Hello %s!' % name) # Function definition # If # While loop # For loop # Basic string # Basic list # Basic dictionaries # Basic sets # Classes # Function definition 2 # List comprehensions # Dict comprehensions # Set comprehensions # Error handling # Generators # Iterators # Decorators # Destructuring assignment # Else clause of loops # f-strings # Collections # Closures # Metaclasses PKjKK7 7 maestro/regex/random_strings.pyimport re import random import functools import string __all__ = ['random_string'] parser = re.sre_compile.sre_parse parse = parser.parse def random_string(pattern): """ Return a random string from the given regex pattern. """ return ''.join(randomizer(pattern)) class RandomChars: """ Provides an iterator that creates a random sequence of characters from a regex. This class is just an implementation detail. Users should stick to the public random_string() function. """ def __init__(self, pattern): self.pattern = pattern if isinstance(pattern, str) else pattern.pattern self.parsed = parse(self.pattern) def __iter__(self): for item in self.parsed: yield from self.yield_from_item(item) def yield_from_item(self, item): head, args = item try: method = getattr(self, 'yield_' + head.name) except AttributeError: raise ValueError('not supported: %s' % head) try: yield from method(args) except: print('head: %s, args: %s' % (head, args)) raise def yield_MAX_REPEAT(self, args): start, end, pattern = args if end is parser.MAXREPEAT: end = start + 256 # Required initial repetitions for _ in range(start): item = random.choice(pattern) yield from self.yield_from_item(item) # Additional repetitions max_size = end - start size = min(int(random.expovariate(0.3)), max_size) size = min(size, 10) for _ in range(size): item = random.choice(pattern) yield from self.yield_from_item(item) def yield_IN(self, items): item = random.choice(items) yield from self.yield_from_item(item) def yield_CATEGORY(self, cat): method = getattr(self, 'random_' + cat.name[9:].lower()) yield str(method()) def yield_RANGE(self, range_): a, b = range_ return chr(random.randrange(a, b)) def yield_LITERAL(self, value): return chr(value) def yield_BRANCH(self, value): arg, patterns = value if arg is not None: raise NotImplementedError('BRANCH: %s' % value) sub_pattern, = random.choice(patterns) yield from self.yield_from_item(sub_pattern) def yield_SUBPATTERN(self, value): group, _, _, data = value for item in data: yield from self.yield_from_item(item) # Randomizers def random_digit(self): return random.randint(0, 9) def random_letter(self): return random.choice(string.ascii_letters) def random_word(self): r = random.random() if r < 0.3: return self.random_digit() elif r < 0.7: return self.random_letter() else: return random.choice(['_']) return random.randint(0, 9) @functools.lru_cache(256) def randomizer(pattern): """ Return an initialized RandomChars instance. """ return RandomChars(pattern) PKfKG--maestro/regex/regex_tester.pyimport re class RegexTesterMeta(type): """ Metaclass for regex-tester classes. """ Re = type.__new__(RegexTesterMeta, (), {'regex': r''}) def make_examples(re_class, accept=5, reject=5): """ Takes a regex class and make a list of accept/reject examples. """ # The teacher class Integer: """ Match any valid positive integer. """ regex = r'[0-9]+' ok = ['42', '0', '1'] bad = ['foo', '41.0'] # For the students class Integer: """ Match any valid positive integer. Good: 42 0 1 Bad: foo 41.0 """ regex = r'' def test_class(cls, data): """ Test a regex class definition. Return None if class was not tested and a tuple (n_ok, n_error) with the number of correct/wrong test cases. """ cls_examples = data[cls.__name__] title = cls.__name__.replace('_', ' ') # Compile regex try: regex = re.compile(cls.regex) except AttributeError: print('%s: class does not define a regex attribute.') return None except Exception: print('%s: invalid regular expression.') return None # Test each suite of examples accept = cls_examples['accept'] reject = cls_examples['reject'] n_ok = 0 msgs = [] for case in accept: if not regex.fullmatch(case): msgs.append('did not match %r.' % case) for case in reject: if regex.fullmatch(case): msgs.append('match %r, but should have rejected it.' % case) # Render message if msgs: print('%s:' % title) print(' correct: %s' % n_ok) print(' wrong:') for msg in msgs: print(' -', msg) else: print('%s: ok!' % title) return n_ok, len(msgs) PKoN%||maestro/test/__init__.pyfrom .checkio import check_interaction, make_spec from .checksecret import check_secret from .checkvars import check_locals PKoNyC C maestro/test/check_locals.pyimport copy import dis from inspect import Signature from types import FunctionType import bytecode from sidekick import placeholder as _, pipe, map from ..utils.exec import safe_exec LOAD_FAST = dis.opmap['LOAD_FAST'] LOAD_NAME = dis.opmap['LOAD_NAME'] STORE_FAST = dis.opmap['STORE_FAST'] STORE_NAME = dis.opmap['STORE_NAME'] DELETE_FAST = dis.opmap['DELETE_FAST'] DELETE_NAME = dis.opmap['DELETE_NAME'] transformer = { LOAD_FAST: LOAD_NAME, STORE_FAST: STORE_NAME, DELETE_FAST: DELETE_NAME, } def check_locals(*args, **_variables): """ Checks if function executed with the given positional arguments defines all local variables passed as keyword arguments. This function does not return anything, but it raises AssertionErrors if some variables have an non-expected value. This function works by introspecting the input function bytecode and comparing the expected values to the value of the local dictionary after execution is completed. Args: func: Function to be executed or a tuple with the function function followed by its positional arguments. Examples: >>> def my_func(x): ... y = x + 1 ... return y >>> check_locals((my_func, 40), y=42) """ func, *kwargs = args kwargs, = kwargs if kwargs else {}, if not callable(func): func: FunctionType func, *args = func arg_names = pipe( Signature.from_callable(func) .parameters.keys(), map(_.name)) kwargs.update(zip(arg_names, args)) code = bytecode.Bytecode(func.__code__) code = fast_to_name(code) code = code.to_code() ns = safe_exec(code, locals=kwargs) for name, value in _variables.items(): assert name in ns, f'Variable {name} not found' assert ns[name] == value, f'Expect: {value!r}, got: {ns[name]!r}' # # Auxiliary functions # def fast_to_name(lst): """ Replaces all *_FAST opcodes into *_NAME codes and disable locals optimization on the resulting bytecode. """ result = copy.copy(lst) result.clear() for instr in lst: code = instr.opcode instr.opcode = transformer.get(code, code) result.append(instr) result.flags = result.flags & (~bytecode.CompilerFlags.OPTIMIZED) return result PK[zN6"Gmaestro/test/checkio.pyimport io import re import sys import typing as typ from collections import deque from collections.abc import Mapping from functools import wraps from ..utils.bultins import update_builtins _print = print _input = input Spec = typ.Union[deque, dict, str] def check_interaction(*args, **kwargs): """ Checks if the given function called with *args and **kwargs produces the expected io. It does nothing in case of success or raises an AssertionError if the main function does not produces the expected io. Args: func (callable): Test function spec: A list of output/input strings. Inputs are represented by sets or lists with a single string entry. Each string entry corresponds to a line of output. Additional positional and keyword arguments are passed to func during execution. Examples: >>> def hello_name(): ... name = input('Name: ') ... print('Hello %s!' % name) >>> spec = [ ... 'Name: ', {'John'}, ... 'Hello John!' ... ] >>> check_interaction(hello_name, spec) """ func, spec, *args = args if isinstance(spec, str): spec = deque([spec]) elif isinstance(spec, Mapping): spec, mapping = deque(), spec.items() for k, v in mapping: spec.extend([k, ignore_ws, v]) else: spec = deque(spec) @wraps(_input) def input_(msg=None): if msg is not None: print_(msg, end='') if not spec: raise AssertionError('Unexpected input') res = spec.popleft(0) if _is_input(res): raise AssertionError('Expects output, but got an input command') # Extract input from singleton list or set try: value, = res return value except TypeError: raise ValueError('Expected input list must have a single value') @wraps(_print) def print_(*args_, file=None, **kwargs_): if file in (None, sys.stdout, sys.stderr): fd = io.StringIO() # noinspection PyTypeChecker _print(*args_, file=fd, **kwargs_) for line in fd.getvalue().splitlines(): _consume_output(line, spec) else: _print(*args_, file=file, **kwargs_) with update_builtins(input=input_, print=print_): func(*args, **kwargs) assert not spec, f'Outputs/inputs were not exhausted: {spec}' def make_spec(func, inputs, *args, **kwargs) -> deque: """ Return the the io spec list that represents an interaction with the given function. Must pass a list of inputs or the Ellipsis (...) object for reading inputs interactively. Examples: >>> hello_name = ['Paul'].__iter__ >>> make_spec(hello_name, ['Paul']) ['Name: ', {'Paul'}, 'Hello Paul!'] See Also: :func:`check_io`: for comparing expected io with the result produced by a function. """ spec = deque() if inputs is not ...: inputs = list(reversed(inputs)) @wraps(_input) def input_(msg=None): if msg is not None: print_(msg, end='') if inputs is ...: value = _input() else: value = inputs.pop() spec.append([value]) return value @wraps(_print) def print_(*args_, file=None, **kwargs_): if file in (None, sys.stdout, sys.stderr): fd = io.StringIO() # noinspection PyTypeChecker _print(*args_, file=fd, **kwargs_) spec.extend(fd.getvalue().splitlines()) if inputs is ...: _print(*args_, **kwargs_) else: _print(*args_, file=file, **kwargs_) with update_builtins(input=input_, print=print_): func(*args, **kwargs) return spec # # Spec creation functions # def ignore_ws(received, spec): """ Consume all whitespace in the beginning of the spec. No-op if first element does not start with whitespace. """ if spec and isinstance(spec[0], str): spec[0] = spec[0].popleft().lstrip() _consume_output(received, spec) def match(regex, test='full'): """ Test if next string matches the given regular expression. """ _re = re.compile(regex) attrs = {'full': 'fullmatch', 'start': 'match'} matches = getattr(_re, attrs[test]) def regex_consumer(received, spec: deque): m = matches(received) if not m: raise AssertionError('Output does not match pattern') if test == 'start': remaining = received[m.end():] if remaining: spec.appendleft(remaining) return regex_consumer # # Auxiliary functions # def _is_input(obj): return not isinstance(obj, str) and not callable(obj) def _consume_output(printed, spec: deque): """ Helper function: consume the given output from io spec. Raises AssertionErrors when it encounter problems. """ if not printed: return elif not spec: raise AssertionError('Asking to consume output, but expects no interaction') elif _is_input(spec[0]): raise AssertionError('Expects input, but trying to print a value') elif printed == spec[0]: spec.popleft() elif callable(spec[0]): spec.popleft()(printed, spec) elif spec[0].startswith(printed): spec[0] = spec[0][len(printed):] elif printed.startswith(spec[0]): n = len(spec.popleft()) _consume_output(printed[n:], spec) else: raise AssertionError(f'Printed wrong value:\n' f' print: {printed!r}\n' f' got: {spec[0]!r}') PK_oN2maestro/test/checksecret.pyimport base64 import sys import zlib from hashlib import blake2b from types import FunctionType import dill from sidekick import pipe def secret(obj, serialize=False): """ Create secret encoding of object. Args: obj: Object to be hidden serialize (bool): If True, uses pickle to serialize object in a way that it can be reconstructed afterwards. Usually this is not necessary and we simply compute a hash of the object. Returns: A string representing the secret representation. """ pickled = dill.dumps(obj) if serialize: data = zlib.compress(pickled) prefix = '$' else: data = blake2b(pickled, digest_size=16) prefix = '#' data = base64.b85decode(data).decode('ascii') return f'{prefix}{data}' def check_secret(obj, secret): """ Executes function serialized as string. Function receives a dictionary with the global namespace and must raise AssertionErrors to signal that a test fails. """ if secret.startswith('#'): assert secret(obj) == secret, 'Objects do not match' else: ref = decode_secret(secret, add_globals=False) assert obj == ref, 'Objects do not match' def decode_secret(secret, *, add_globals=True): """ Extract object from secret. Args: secret: String containing encoded secret. add_globals (bool): For functions encoded using a secret, it adds the globals dictionary to function's own __code__.co_globals. """ if not secret.startswith('$'): raise ValueError('not a valid secret') obj = pipe( secret[1:].encode('ascii'), base64.b85decode, zlib.decompress, dill.loads, ) if add_globals is not False and isinstance(obj, FunctionType): frame = sys._getframe(2 if add_globals is True else add_globals) obj.__code__.co_globals = frame.f_globals return obj PK'QCOmaestro/test/testfunc.pyimport operator as op from types import MappingProxyType from sidekick import Result, itertools from ..utils.text import safe_repr, make_string def check_sync(f1, f2, test_cases, check_all=False, kwargs=False, simeq=False): """ Checks if function f1 and f2 produces the same output for all the given inputs Args: f1, f2: Functions that should be compared. In error messages, f1 is treated as the source of truth and f2 as the test function. test_cases (list): List of arguments to apply to f1 and f2. Tuple arguments are expanded as multiple arguments check_all (bool): If True, collect the list of all errors and raise an exception which displays all problems found. kwargs (bool): If True, assumes that args is a list of args tuples and kwargs dictionaries. Any of the two elements may be missing. simeq (bool, callable): If True, accepts similar, but not identical results. This is generally a good idea when testing floating point outputs. It can also be a callable that receives each result and test them for approximate equality. Examples: >>> from math import tan, sin, pi >>> check_sync(tan, sin, [0, pi, 2*pi], simeq=True) # Pass! >>> check_sync(tan, sin, [0, pi/2, pi, 3/2 * pi, 2*pi], simeq=True) # Fails! Raises: Raises an AssertionError if any two outputs are different. """ if simeq is False: simeq = op.eq errors = [] for args, kwargs in map(normalize_args(kwargs), test_cases): res1 = Result.call(f1, *args, **kwargs) res2 = Result.call(f2, *args, **kwargs) error = Result.first_error(res2, res1) if error is None: cmp = Result.apply(simeq, res1, res2) if cmp.is_err: error = cmp.error elif cmp.value is False: expect = safe_repr(res1.value, 20) got = safe_repr(res2.value, 20) args = format_call_args(args, kwargs, safe_repr) error = f'for f({args}), expect: {expect}, got: {got}' if error is not None: if check_all: errors.append(error) else: raise AssertionError(format_error(error)) if check_all and errors: msg = '\n'.join(map(line_prefix(' * '), map(format_error, errors))) raise AssertionError('Many errors were found:\n', msg) def normalize_args(kwargs): def normalizer(x) -> (tuple, dict): if kwargs: if isinstance(x, tuple): if (len(x) == 2 and isinstance(x[0], tuple) and isinstance(x[1], dict)): return x return x, {} elif isinstance(x, tuple): return x, {} else: return (x,), {} return normalizer def format_error(err) -> str: if isinstance(err, str): return err else: cls = type(err).__name__ msg = str(err) return f'{cls}: {msg}' def format_call_args(args=(), kwargs=MappingProxyType({}), repr=repr): """ Format the args and kwargs of a function signature. using safe_repr """ return ', '.join(itertools.chain( map(repr, args), (f'{k}={repr(v)}' for k, v in kwargs.items()) )) # # String formatting (move to sidekick) # CURRIED = object() def line_prefix(item, st: str = CURRIED): """ Add prefix to the first line of string. Following lines are indented to the number of spaces equal to the prefix. """ if st is CURRIED: return lambda x: line_prefix(item, x) size = len(item) return item + indent(size, st, skip_first=True) def indent(size, st: str = CURRIED, skip_first=False): """ Indent text with the given indent. Indent can be a string or a number that represents the number of spaces that should prefix each line. """ if st is CURRIED: return lambda x: indent(size, x, skip_first) pre = ' ' * size if isinstance(size, int) else make_string(size) lines = iter(st.splitlines()) prefix = (lambda x: pre + x) if skip_first: first = next(lines) lines = itertools.chain([first], map(prefix, lines)) else: lines = map(prefix, lines) return '\n'.join(lines) PKK\A maestro/types/__init__.pyfrom .numbers import Int, Float PKxKmaestro/types/base.pyPKKcmaestro/types/examples.pyimport random # # Examples # class ExamplesMeta(type): def __getitem__(cls, item): meta = cls.__class__ return meta(cls.__name__, (cls,), {'examples': item}) def __instancecheck__(cls, instance): return instance in cls.examples class RandomExamplesMixin: "Adds the random() method to class" examples = [] @classmethod def random(cls): return random.choice(cls.examples) class Examples(RandomExamplesMixin, metaclass=ExamplesMeta): """ A value from a set of examples. Usage: >>> examples = [1, 2, 3] >>> Examples[examples].random() in [1, 2, 3] True """ # # External data sources # class LineOfMeta(ExamplesMeta): def __getitem__(cls, path): with open(path) as F: lines = F.readlines() examples = [line.rstrip('\n') for line in lines] new = super().__getitem__(examples) new.path = path return new class LineOf(RandomExamplesMixin, metaclass=LineOfMeta): """ Examples are stored in the given file path. Given a file with the contents:: foo bar We obtain random values from >>> LineOf['examples.test'].random() in ['foo', 'bar'] """ PKPCO6FFmaestro/types/numbers.pyimport random MAX_INT = 2 ** 65 class NumberMeta(type): def __getitem__(cls, idx): a, b = idx if a < b: raise ValueError('first index should be smaller than second!') ns = dict(minimum=a, maximum=b) return NumberMeta(cls.__name__, (cls,), ns) def __instancecheck__(cls, instance): return isinstance(instance, cls._type) class NumberBase(metclass=NumberMeta): """Base class for Int and Float""" minimum = ... maximum = ... class Int(NumberBase): """A random integral number""" _type = int @classmethod def random(cls): min, max = cls.minimum, cls.maximum min = -MAX_INT if min is ... else min max = MAX_INT if max is ... else max return random.randint(min, max) class Float(NumberBase): _type = float PKuMmaestro/utils/__init__.pyPKA`oN,55maestro/utils/bultins.pyimport builtins from contextlib import contextmanager from sidekick import record @contextmanager def update_builtins(**kwargs): """ Context manager that temporarily sets the specified builtins to the given values. Examples: >>> with update_builtins(print=lambda *args: None) as orig: ... print('Hello!') # print is shadowed here ... orig.print('Hello!') # Calls real print """ undefined = object() revert = {k: getattr(builtins, k, undefined) for k in kwargs} try: for k, v in kwargs.items(): setattr(builtins, k, v) yield record(**{k: v for k, v in revert.items() if v is not undefined}) finally: for k, v in revert.items(): if v is not undefined: setattr(builtins, k, v) PKX uMymaestro/utils/exec.pyimport threading def safe_exec(code, globs=None, locals=None, timeout=None): """ Similar to the builtin exec() function, but has some security measures. Differently from the builtin, it returns the resulting timeout. Args: code: Code, or source object. globs, locals: Optional dictionaries of globals and locals. timeout (float): Optional timeout in seconds. """ if globs is None: globs = {} if locals is None: locals = globs if timeout is not None: with_timeout(timeout, exec, code, globs, locals) else: exec(code, globs, locals) return locals def with_timeout(*args, **kwargs): """ Exec function with the given timeout Args: timeout: Timeout in seconds. func: Function to be executed. Examples: >>> with_timeout(5, sum, [1, 2, 3]) 6 """ timeout, func, *args = args result_container = [] def wrapped(): result_container.append(func(*args, **kwargs)) thread = threading.Thread(target=wrapped) thread.start() thread.join(timeout) if result_container: return result_container[0] raise ValueError('function could not produce output') PKPIpN#C33maestro/utils/text.pydef safe_repr(x, truncate=None): """ Like repr(), but never raises exceptions. It provides a sane default in the problematic cases. """ try: return truncate(repr(x), x) except Exception as ex: ex_name = getattr(type(ex), '__name__', 'Exception') try: msg = str(ex) except: msg = 'invalid error message' else: msg = truncate_line(msg, 40) cls = type(x) cls_name = getattr(cls, '__name__', 'object') return f'<{cls_name} instance at {hex(id(x))} [{ex_name}: {msg}]>' def truncate(st, size=80): """ Truncate string to the maximum given size. """ if size is None: return st if len(st) > size: return st[:size - 3] + '...' return st def truncate_line(st, size=80): """ Truncate string to the first non-empty line and with the maximum given size. """ return truncate(st.lstrip().splitlines()[0], size) def make_string(x): """ Force object into string and raises a TypeError if object is not of a compatible type. """ if isinstance(x, str): return str elif isinstance(x, bytes): return x.decode('utf8') else: cls_name = type(x).__name__ raise TypeError(f'expect string type, got {cls_name}')PK!H%5(,.maestro_tools-0.1.0.dist-info/entry_points.txtN+I/N.,()ML-.)ʷz9Vy\\PK^sNC@@%maestro_tools-0.1.0.dist-info/LICENSEThe MIT License (MIT) Copyright (c) 2019 Fábio Macêdo Mendes Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!HMuSa#maestro_tools-0.1.0.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,szd&Y)r$[)T&UD"PK!H5&maestro_tools-0.1.0.dist-info/METADATAeMN0>\ IS,Q4(?=Mqn [ы1i viPcB40KQ# )位e>9 [ &֒J́_/ѐU`Z肯:g:[a$ ;:yoR'6WXxcB<EmdkX-+1Sツ7L~4ϽfX HT!aYL;Ʌ?>z(fw&& 6qIDT֨͠]kͶ/5o]2h  || -Z;+~PK!H 2 $maestro_tools-0.1.0.dist-info/RECORDGX}&E/ a`׏絟J%MƒY|{ !ƾ3[FbVoI}piY6yxN[(mqcKiE.* (^"Tp4{ ,͕=]+bTqu6kfrQvw%2DAJ>K6iQi#qNprִQz]ʼnV\c^ּQ &z68j:nz+ep0`2D1NcQek\iKL}CTse#=aS?nf|)[o R5Gq>Q"8<̰{OŁh:.nX{q>zk2 ]YCB07r0,mBCb(TbWjbC=} A@/x,g/`_,6n:Œ鹲l9K%QCW]͊JgVͪ{x\zcc޼6+0'VB4{~$O7ݟ;PPhE=럝K'E&γ<=pLUGr19Q~w\|)2N4z]'c4D MR p';v QK_`h sJr^='z'JK.̬1 .]!ÓO)nۧ],s_hѾx#WAf[*l%W\%wnqFr" @܃uECQ0 C$O&dQ5*fF&g >s8<{"rKg/90ujZ_s};bI4dHaB'7jwTv(!ϙEOOGp4cHcLgh ipN le ȹW{ob3s3ɘAkZ]VQWx+z\4UcѼ]Y' K:3&9.^gc C