commit 73d3e0168de01d86175de7f4f6314f6ec1f5be72 parent b78fd7280b2b0d57e186546516d1ce8f3dbcc324 Author: Fabian Wermelinger <fab4100@posteo.net> Date: Thu, 15 Aug 2024 17:08:49 +0200 Add lecture codes Diffstat:
172 files changed, 17140 insertions(+), 0 deletions(-)
diff --git a/lecture03/redirections.sh b/lecture03/redirections.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# File : redirections.sh +# Description: Some redirection examples for stdout and stderr +# Copyright 2022 Harvard University. All Rights Reserved. + +echo 'The `echo` command outputs to stdout:' +echo 'Hello CS107/AC207' +echo 'Hello CS107/AC207' >/dev/null # no output to stdout here + +echo 'The `cp` command takes at least two arguments:' + +cp >out1 # this will fail and print an error message on your screen. The stdout + # stream will be empty and so is the file out1 + +cp 2>/dev/null # this will redirect the stderr stream into /dev/null. You will + # not see the error message on your screen + +cp >out2 2>&1 # this will redirect both stdout and stderr to the file out2. + # There will be no output on the screen diff --git a/lecture03/shell_scripting/file_check.sh b/lecture03/shell_scripting/file_check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# File : file_check.sh +# Description: Test type of file passed as argument to the script +# Copyright 2022 Harvard University. All Rights Reserved. + +if [ $# -ne 1 ]; then + cat <<EOF +USAGE: $0 <path/to/file> + + More documentation here. The form used here is called a here-document. + They are very useful to write longer strings and expanding variables like + \$0 above. See https://tldp.org/LDP/abs/html/here-docs.html +EOF + exit 1 # exit with failure code +fi + +if [ -f $1 ]; then + echo "File $1 exists and is a regular file" +elif [ -d $1 ]; then + echo "File $1 exists and is a directory" +elif [ -e $1 ]; then + # any other file type (character, block or symbolic link) + echo "File $1 exists and is an unknown file" +fi diff --git a/lecture03/shell_scripting/if_conditionals.sh b/lecture03/shell_scripting/if_conditionals.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# File : int_comparison.sh +# Description: Some string comparison examples +# Copyright 2022 Harvard University. All Rights Reserved. + +if [ 'abc' == 'abc' ]; then + echo "'abc' == 'abc'" +fi + +str='' # empty string +if [ -z $str ]; then + echo 'str is empty' +fi + +if [ ! -z $str ]; then # negation + echo 'str is not empty' +else + echo 'str is empty' +fi + +str='abc' +if [ -n $str ]; then # alternative: check if string is non-empty + echo 'str is not empty' +fi diff --git a/lecture03/shell_scripting/int_comparison.sh b/lecture03/shell_scripting/int_comparison.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# File : int_comparison.sh +# Description: Integer comparison based on number of arguments given to the +# script +# Copyright 2022 Harvard University. All Rights Reserved. + +if [ $# -gt 2 ]; then + echo "Number of arguments $# is larger than two" +else + echo "Number of arguments $# is less than or equal to two" +fi diff --git a/lecture03/shell_scripting/my_exec b/lecture03/shell_scripting/my_exec @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# File : my_exec +# Description: Shell script example from lecture slides +# Copyright 2022 Harvard University. All Rights Reserved. + +echo "Script $0 running inside $PWD" +echo "The following arguments were given:" +for arg in "$@"; do + echo $arg +done +exit 0 diff --git a/lecture03/shell_scripting/string_comparison.sh b/lecture03/shell_scripting/string_comparison.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# File : string_comparison.sh +# Description: Example for comparing string argument +# Copyright 2022 Harvard University. All Rights Reserved. + +if [ "$1" == 'Hello CS107/AC207!' ]; then + echo 'Success!' +else + echo 'Got unexpected string argument' +fi diff --git a/lecture03/this_command_exists_locally b/lecture03/this_command_exists_locally @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# File : this_command_exists_locally +# Description: Simple scripted command (not in PATH environment variable) +# Copyright 2022 Harvard University. All Rights Reserved. +echo "You have executed $0" +exit 0 diff --git a/lecture04/diff/.gitignore b/lecture04/diff/.gitignore @@ -0,0 +1 @@ +fix.patch diff --git a/lecture04/diff/README.md b/lecture04/diff/README.md @@ -0,0 +1,12 @@ +# Steps to patch file `branch_B/hello.py` + +1. Create a patch file from a diff: +```bash +diff --color=never -u branch_B/hello.py branch_A/hello.py >fix.patch +``` +2. Change into `branch_B/` and apply the patch to the file `hello.py`. Check + the contents of the file *before* you apply the patch and once more after. + To apply the patch: +```bash +patch -p1 <../fix.patch +``` diff --git a/lecture04/diff/branch_A/hello.py b/lecture04/diff/branch_A/hello.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# File : hello.py +# Description: Simple Python code with function +# Copyright 2022 Harvard University. All Rights Reserved. + + +def main(): + print('Hello CS107/AC207') + + +if __name__ == "__main__": + main() diff --git a/lecture04/diff/branch_B/hello.py b/lecture04/diff/branch_B/hello.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# File : hello.py +# Description: Simple Python code with function +# Copyright 2022 Harvard University. All Rights Reserved. + + +def main(): + print('Hello CS107') + + +if __name__ == "__main__": + main() diff --git a/lecture04/git_interactive_rebase/.gitignore b/lecture04/git_interactive_rebase/.gitignore @@ -0,0 +1 @@ +/git diff --git a/lecture04/git_interactive_rebase/setup.sh b/lecture04/git_interactive_rebase/setup.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +rm -rf git +mkdir -p git/repo +mkdir -p git/remote + +# initialize remote +(cd git/remote && git init --bare) + +# initialize repository +cd git/repo +git init +git config user.name 'Dev Eloper' +git config user.email 'dev@eloper.org' +git remote add origin ../remote +git branch -M main + +# create content +echo 'Initial' >file +git add file +git commit -m 'Initial' +git push -u origin main + +# create modifications +for (( i = 1; i < 11; i++ )); do + echo "Modification ${i}" >>file + + # create commit + git add file + git commit -m "Modification ${i}" +done diff --git a/lecture04/git_objects/.gitignore b/lecture04/git_objects/.gitignore @@ -0,0 +1 @@ +/repo diff --git a/lecture04/git_objects/setup.sh b/lecture04/git_objects/setup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +rm -rf repo +mkdir -p repo + +# initialize repository +cd repo +git init +git config user.name 'Dev Eloper' +git config user.email 'dev@eloper.org' + +# create content +echo 'Hello CS107/AC207!' >hello.txt + +# create commit +git add hello.txt +git commit -m 'Initial' diff --git a/lecture05/git_remote/.gitignore b/lecture05/git_remote/.gitignore @@ -0,0 +1 @@ +/git diff --git a/lecture05/git_remote/setup.sh b/lecture05/git_remote/setup.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# File : setup.sh +# Description: Setup example Git repositories with a remote +# Copyright 2022 Harvard University. All Rights Reserved. +rm -rf git +mkdir -p git/remote + +# initialize remote (bare repository) +(cd git/remote && git init --bare) + +# initialize A +mkdir -p git/A +cd git/A +git init +git config user.name 'Developer A' +git config user.email 'A@domain.org' +git branch -M main + +# setup the remote and create content +git remote add origin ../remote # no URL this time +echo 'Initial' >file +git add file +git commit -m 'Initial' +git push -u origin main + +# B clones +cd .. # inside `git` directory +git clone remote B +cd B # inside repository B +git config user.name 'Developer B' +git config user.email 'B@domain.org' +git config branch.main.rebase 'false' +# default branch name already defined by A diff --git a/lecture06/notebooks/python_1.ipynb b/lecture06/notebooks/python_1.ipynb @@ -0,0 +1,1335 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introductory Python" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The main topic for today's lecture is Python and some of it's basic\n", + "functionality. We will cover the basics of \n", + "\n", + "* using Python as a calculator\n", + "* `print` statements\n", + "* the list concept\n", + "* opening and reading from files\n", + "* dictionaries\n", + "* strings\n", + "\n", + "I will show you some very basic examples and you will put them all together in a\n", + "small script for your exercise. The exercise is displayed at the top of this\n", + "notebook. If you already know how to do it, then just write up your script now.\n", + "However, you may need some guidance. You will find such guidance throughout the\n", + "rest of the notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Important, Useful Libraries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should always try to use existing technologies to accomplish your goals\n", + "whenever possible. For example, don't write your own function to compute the\n", + "square root of a number. That would be really hard and your implementation\n", + "would most likely not be very efficient. Instead, use built-in functionality or\n", + "functionality from a nice library such as `numpy`\n", + "([NUMericalPYthon](http://www.numpy.org/)).\n", + "\n", + "> NumPy is the fundamental package for scientific computing with Python. It\n", + "> contains among other things:\n", + ">\n", + "> * a powerful N-dimensional array object \n", + "> * sophisticated (broadcasting) functions \n", + "> * tools for integrating C/C++ and Fortran code \n", + "> * useful linear algebra, Fourier transform, and random number capabilities \n", + ">\n", + "> Besides its obvious scientific uses, NumPy can also be used as an efficient\n", + "> multi-dimensional container of generic data. Arbitrary data-types can be\n", + "> defined. This allows NumPy to seamlessly and speedily integrate with a wide\n", + "> variety of databases.\n", + "\n", + "To import libraries into your Python application, do the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# The %... is an iPython thing, and is not part of the Python language.\n", + "# In this case we're just telling the plotting library to draw things on\n", + "# the notebook, instead of on a separate window.\n", + "%matplotlib inline \n", + "# the line above prepares IPython notebook for working with matplotlib\n", + "\n", + "import numpy as np # imports a fast numerical programming library\n", + "import scipy as sp #imports stats functions, amongst other things\n", + "import matplotlib as mpl # this actually imports matplotlib\n", + "import matplotlib.cm as cm #allows us easy access to colormaps\n", + "import matplotlib.pyplot as plt #sets up plotting under plt\n", + "import pandas as pd #lets us handle data as dataframes\n", + "#sets up pandas table display\n", + "pd.set_option('display.width', 500)\n", + "pd.set_option('display.max_columns', 100)\n", + "pd.set_option('display.notebook_repr_html', True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The way to understand these imports is as follows: _import the library `library`\n", + "with the alias `lib`_ where `library` could be `numpy` or `matplotlib` or\n", + "whatever you want and `lib` is the alias used to refer to that library in our\n", + "code. Using this flow, we can call methods like `plt.plot()` instead of\n", + "`matplotlib.pyplot.plot()`. It makes life easier." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**NOTE:** It is not necessary to import _all_ of these libraries all of the\n", + "time. You should only import the ones you really need. I listed a bunch above\n", + "to give you a sampling of what's available.\n", + "\n", + "**NOTE:** DO NOT include `%matplotlib inline` in your Python scripts unless\n", + "you're working in the Jupyter notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At the end of this course, someone should be able to `import\n", + "your_kinetics_library` to use the kinetics library that you are about to start\n", + "writing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Very Basics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll fly through this part because you should already know it. If you don't\n", + "understand something, please Google it and/or refer to the [Python\n", + "Tutorial](https://docs.python.org/3/tutorial/). I do not want to recreate the\n", + "Python tutorial here; instead, I'll just summarize a few important ideas from\n", + "Python. We'll give more details a little later on how some of these language\n", + "features work.\n", + "\n", + "Another very helpful resource that explains the basics below (and few additional\n", + "topics) can be found here:\n", + "[https://learnxinyminutes.com/docs/python/](https://learnxinyminutes.com/docs/python/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Calculating" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can tell the type of a number or variable by using the `type` function." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(int, float)" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(3), type(3.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remember, every variable in python gets a type. Python is a strongly typed\n", + "language. It is also a dynamic language, in the sense that types are assigned at\n", + "run-time, rather then \"compile\" time, as in a language like C. This makes it\n", + "slower, as the way data is stored cannot be initially optimal, as when the\n", + "program starts, you dont know what that variable will point to." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All the usual calculations can be done in Python." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6.0" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "2.0 + 4.0 # Adding two floats" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "2 + 4 # Adding two ints" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3333333333333333" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1.0 / 3.0 # Dividing two floats" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3333333333333333" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1 / 3 # Dividing two ints" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that in Python 2, the division of two ints would not be interpreted as a\n", + "float; it is integer division. This is new in Python 3! Now, if you want\n", + "integer division you have to use the `//` operator." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1 // 3 # Integer division" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "32" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "2**5 # Powers" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "15" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "3 * 5 # Multiplication" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### More advanced operations\n", + "\n", + "We can use `numpy` to do some more advanced operations." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "13.974998513319154" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.pi * np.exp(2.0) + np.tanh(1.0) - np.sqrt(100.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that I am always writing my floats with a decimal point. You don't\n", + "really need to do that in Python because Python will automatically convert\n", + "between types. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(numpy.float64, numpy.float64)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "type(np.pi * np.exp(2.0) + np.tanh(1.0) - np.sqrt(100.0)), type(np.pi * np.exp(2) + np.tanh(1) - np.sqrt(100))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, I like to make the types as explicit as I can so there's no confusion." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### `print`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `print` function is the basic way to write information out to the screen. I\n", + "will briefly review the new form of the `print` function. In Python 2, `print`\n", + "was a `statement` rather than a `function`." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Good morning! Today we are doing Python!\n", + "3.0\n", + "3.141592653589793 is a nice, trancendental number\n", + "Eric is nice and so is Sarah\n", + " 3.1415926535897931...: it goes on forever but 3 is just an int.\n" + ] + } + ], + "source": [ + "print('Good morning! Today we are doing Python!') # Basic print\n", + "print(3.0) # Print a float\n", + "print('{} is a nice, trancendental number'.format(np.pi)) # Print just one number\n", + "print('{} is nice and so is {}'.format('Eric', 'Sarah')) # Print with two arguments\n", + "print('{0:20.16f}...: it goes on forever but {1} is just an int.'.format(np.pi, 3)) # Print with formatting in argument 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are some additional resources for the `print` function and formatting:\n", + "* [7. Input and Output](https://docs.python.org/3/tutorial/inputoutput.html)\n", + "* [Formatted Output](https://www.python-course.eu/python3_formatted_output.php)\n", + "* [`Print` function](https://docs.python.org/3/library/functions.html#print)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Variables" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll have more to say about variables in Python later. For now, here's how you\n", + "store them syntactically:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.0x^2 + -1.0x + -1.0 = 0.0\n" + ] + } + ], + "source": [ + "a = 1.0\n", + "b = -1.0\n", + "c = -1.0\n", + "x = (1.0 + np.sqrt(5.0)) / 2.0\n", + "val = a * x**2.0 + b * x + c\n", + "print('{0}x^2 + {1}x + {2} = {3}'.format(a, b, c, val))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Python has this nice feature where you can assign more than one variable all on\n", + "one line. It's called the multiple assignment statement." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.0x^2 + -1.0x + -1.0 = 0.0\n" + ] + } + ], + "source": [ + "a, b, c = 1.0, -1.0, -1.0\n", + "x = (1.0 + np.sqrt(5.0)) / 2.0\n", + "val = a * x**2.0 + b * x + c\n", + "print('{0}x^2 + {1}x + {2} = {3}'.format(a, b, c, val))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Looks a little cleaner now." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Lists and `for` loops" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lists are central to Python. Many things behave like lists. For now, we'll\n", + "just look at how to create them and do basic operations with them. I will not\n", + "go through all the details. Please refer to\n", + "[Lists](https://docs.python.org/3/tutorial/introduction.html#lists) for\n", + "additional examples." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First few primes are: [2, 3, 5, 7, 11, 13]\n", + "Here are the primes up to the number 20: [2, 3, 5, 7, 11, 13, 17, 19]\n" + ] + } + ], + "source": [ + "primes = [2, 3, 5, 7, 11, 13] # A list of primes\n", + "more_primes = primes + [17, 19] # List concatentation\n", + "print('First few primes are: {primes}'.format(primes=primes))\n", + "print('Here are the primes up to the number 20: {}'.format(more_primes))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that Python knows that type of `primes`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "primes is of type <class 'list'>\n" + ] + } + ], + "source": [ + "print('primes is of type {}'.format(type(primes)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `len` function can provide the number of elements in the list." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "There are 6 prime numbers less than or equal to 20.\n" + ] + } + ], + "source": [ + "print('There are {} prime numbers less than or equal to 20.'.format(len(primes)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we know what a list is, we can discuss `for` loops in Python. The\n", + "`for` loop iterates over an iterator such as a list. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2\n", + "3\n", + "5\n", + "7\n", + "11\n", + "13\n", + "17\n", + "19\n" + ] + } + ], + "source": [ + "for p in more_primes:\n", + " print(p)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A useful iterator (but not a list!) is the `range` function." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "range(0, 10)\n", + "<class 'range'>\n" + ] + } + ], + "source": [ + "print(range(10))\n", + "print(type(range(10)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's not a list anymore (it used to be in Python 2) and therefore can't be\n", + "sliced like a list can (see below). Still, you can use it in `for` loops which\n", + "is where it finds most of its use." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n", + "1\n", + "2\n", + "3\n", + "4\n", + "5\n", + "6\n", + "7\n", + "8\n", + "9\n" + ] + } + ], + "source": [ + "for n in range(10):\n", + " print(n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There is something called a _list comprehension_ in Python. List comprehensions\n", + "are just a way to transform one list into another list." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The new list is [0, 1, 1, 2, 3, 4, 5, 6]\n" + ] + } + ], + "source": [ + "not_all_primes = [p // 3 for p in more_primes]\n", + "print('The new list is {}'.format(not_all_primes))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also count the number of each element in the list. There are a number of\n", + "ways of doing this, but one convenient way is to use the `collections` library." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Counter({1: 2, 0: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1})\n", + "<class 'collections.Counter'>\n" + ] + } + ], + "source": [ + "import collections\n", + "how_many = collections.Counter(not_all_primes)\n", + "print(how_many)\n", + "print(type(how_many))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that there are 2 ones, 1 two, 1 three, etc.\n", + "\n", + "We can even find the most common element of the list and how many occurrences of\n", + "it there are and return the result as a list." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[(1, 2), (0, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)]\n", + "<class 'list'>\n" + ] + } + ], + "source": [ + "how_many_list = how_many.most_common()\n", + "print(how_many_list)\n", + "print(type(how_many_list))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that the result is a list of tuples with the most common element of our\n", + "original list (`not_all_primes`) displayed first. We want the most common\n", + "element of our original list, so we just access the first element using a simple\n", + "index." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(1, 2)\n", + "<class 'tuple'>\n" + ] + } + ], + "source": [ + "most_common = how_many_list[0]\n", + "print(most_common)\n", + "print(type(most_common))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're almost there. We recall the first element of this tuple is the value from\n", + "our original list and the second element in the tuple is its frequency. We're\n", + "finally ready to get our result!" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The number 1 is the most common value in our list.\n", + "It occurs 2 times.\n" + ] + } + ], + "source": [ + "print('The number {} is the most common value in our list.'.format(most_common[0]))\n", + "print('It occurs {} times.'.format(most_common[1]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List indexing is also very important. It can also do much more than what we did\n", + "above." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5\n", + "[5, 7, 11]\n", + "13\n", + "[7, 11, 13]\n" + ] + } + ], + "source": [ + "print(primes[2]) # print the 3rd entry \n", + "print(primes[2:5]) # print the 3rd to 5th entries\n", + "print(primes[-1]) # print the last entry\n", + "print(primes[-3:]) # print the three entries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Other types of slices and indexing can be done as well. I leave it to you to\n", + "look this up as you need it. It is a **very** useful thing to know." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Two convenient built-in functions are `enumerate` and `zip`. You may find\n", + "various uses for them.\n", + "\n", + "* `enumerate` gives a representation of a list of tuples with each tuple of the\n", + " form `(index, value)`. This provides an easy way to access the `index` of the\n", + " value in the `list`.\n", + "* `zip` takes elements from each list and puts them together into a\n", + " representation of a list of tuples. This provides a nice way to aggregate\n", + " lists." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll make two lists for the following examples:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "species = ['H2', 'O2', 'OH', 'H2O', 'H2O2']\n", + "species_names = ['Hydrogen', 'Oxygen', 'Hydroxyl', 'Water', 'Hydrogen Peroxide']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### `enumerate` example" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<enumerate object at 0x10adecb40>\n" + ] + } + ], + "source": [ + "print(enumerate(species)) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that `enumerate()` just returns an iterator object. To actually see\n", + "what's in the iterator object, we need to convert the iterator object to a list" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[(0, 'H2'), (1, 'O2'), (2, 'OH'), (3, 'H2O'), (4, 'H2O2')]\n" + ] + } + ], + "source": [ + "print(list(enumerate(species)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that we have a list of tuples (in the form `(index, value)` where `index`\n", + "starts from 0). Here's just one way that this might be used:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "H2 is species 1\n", + "O2 is species 2\n", + "OH is species 3\n", + "H2O is species 4\n", + "H2O2 is species 5\n" + ] + } + ], + "source": [ + "for i, s in enumerate(species):\n", + " print('{species} is species {ind}'.format(species=s, ind=i+1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What happened is that the `for` loop iterated over the iterable (here\n", + "`enumerate`). The first index in the `for` loop corresponds to the first entry\n", + "in the `enumerate` tuple and the second index in the `for` loop corresponds to\n", + "the second entry in the `enumerate` tuple." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### `zip` example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see how `zip` works. We'll aggregate the `species` and `species_names`\n", + "lists." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<zip object at 0x10adfb108>\n", + "[('H2', 'Hydrogen'), ('O2', 'Oxygen'), ('OH', 'Hydroxyl'), ('H2O', 'Water'), ('H2O2', 'Hydrogen Peroxide')]\n" + ] + } + ], + "source": [ + "print(zip(species, species_names))\n", + "print(list(zip(species, species_names)))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "H2 is called Hydrogen\n", + "O2 is called Oxygen\n", + "OH is called Hydroxyl\n", + "H2O is called Water\n", + "H2O2 is called Hydrogen Peroxide\n" + ] + } + ], + "source": [ + "for s, name in zip(species, species_names):\n", + " print('{specie} is called {name}'.format(specie=s, name=name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that this worked in a similar way to `enumerate`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, you will sometimes see `enumerate` and `zip` used together." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Species 1 is H2 and it is called Hydrogen.\n", + "Species 2 is O2 and it is called Oxygen.\n", + "Species 3 is OH and it is called Hydroxyl.\n", + "Species 4 is H2O and it is called Water.\n", + "Species 5 is H2O2 and it is called Hydrogen Peroxide.\n" + ] + } + ], + "source": [ + "for n, (s, name) in enumerate(zip(species, species_names), 1):\n", + " print('Species {ind} is {specie} and it is called {name}.'.format(ind=n, specie=s, name=name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Opening Files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are a variety of ways to open files in Python. We'll see a bunch as the\n", + "semester progresses. Today, we'll focus on opening and reading text files." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "species_file = open(\"species.txt\") # Open the file\n", + "species_text = species_file.read() # Read the lines of the file\n", + "species_tokens = species_text.split() # Split the string and separate based on white spaces\n", + "species_file.close() # Close the file!" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['H2', 'O2', 'OH', 'H2O', 'H2O2']\n", + "<class 'list'>\n" + ] + } + ], + "source": [ + "print(species_tokens)\n", + "print(type(species_tokens))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that we get a list of strings." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's a better way to open a file. The `close` operation is handled\n", + "automatically for us." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open('species.txt') as species_file:\n", + " species_text = species_file.read()\n", + " species_tokens = species_text.split()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dictionaries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Dictionaries are extremely important in Python. For particular details on\n", + "dictionaries refer to\n", + "[Dictionaries](https://docs.python.org/3/tutorial/datastructures.html#dictionaries).\n", + "From that tutorial we have a few comments on dictionaries:\n", + "\n", + "> Unlike sequences, which are indexed by a range of numbers, dictionaries are\n", + "> indexed by keys, which can be any immutable type; strings and numbers can\n", + "> always be keys.\n", + ">\n", + "> It is best to think of a dictionary as an unordered set of key: value pairs,\n", + "> with the requirement that the keys are unique (within one dictionary). A pair\n", + "> of braces creates an empty dictionary: {}. Placing a comma-separated list of\n", + "> key:value pairs within the braces adds initial key:value pairs to the\n", + "> dictionary; this is also the way dictionaries are written on output.\n", + ">\n", + "> The main operations on a dictionary are storing a value with some key and\n", + "> extracting the value given the key." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create a chemical species dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'H2': 'Hydrogen', 'O2': 'Oxygen', 'OH': 'Hydroxyl', 'H2O': 'Water', 'H2O2': 'Hydrogen Peroxide'}\n" + ] + } + ], + "source": [ + "species_dict = {'H2':'Hydrogen', 'O2':'Oxygen', 'OH':'Hydroxyl', 'H2O':'Water', 'H2O2':'Hydrogen Peroxide'}\n", + "print(species_dict)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The entries to the left of the colon are the keys and the entries to the right\n", + "of the colon are the values. To access a value we just reference the key." + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hydrogen\n" + ] + } + ], + "source": [ + "print(species_dict['H2'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pretty cool!\n", + "\n", + "Suppose we want to add another species to our dictionary. No problem!" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'H2': 'Hydrogen', 'O2': 'Oxygen', 'OH': 'Hydroxyl', 'H2O': 'Water', 'H2O2': 'Hydrogen Peroxide', 'H': 'Atomic Hydrogen'}\n", + "Atomic Hydrogen\n" + ] + } + ], + "source": [ + "species_dict['H'] = 'Atomic Hydrogen'\n", + "print(species_dict)\n", + "print(species_dict['H'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Why should we use dictionaries at all? Clearly they're very convenient. But\n", + "they're also fast. See [indexnext |previous |How to Think Like a Computer\n", + "Scientist: Learning with Python 3: 20.\n", + "Dictionaries](http://openbookproject.net/thinkcs/python/english3e/dictionaries.html)\n", + "for a decent explanation." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,md" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/lecture06/notebooks/python_1.md b/lecture06/notebooks/python_1.md @@ -0,0 +1,499 @@ +--- +jupyter: + jupytext: + formats: ipynb,md + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 + language: python + name: python3 +--- + +# Introductory Python + + +The main topic for today's lecture is Python and some of it's basic +functionality. We will cover the basics of + +* using Python as a calculator +* `print` statements +* the list concept +* opening and reading from files +* dictionaries +* strings + +I will show you some very basic examples and you will put them all together in a +small script for your exercise. The exercise is displayed at the top of this +notebook. If you already know how to do it, then just write up your script now. +However, you may need some guidance. You will find such guidance throughout the +rest of the notebook. + + +## Important, Useful Libraries + + +You should always try to use existing technologies to accomplish your goals +whenever possible. For example, don't write your own function to compute the +square root of a number. That would be really hard and your implementation +would most likely not be very efficient. Instead, use built-in functionality or +functionality from a nice library such as `numpy` +([NUMericalPYthon](http://www.numpy.org/)). + +> NumPy is the fundamental package for scientific computing with Python. It +> contains among other things: +> +> * a powerful N-dimensional array object +> * sophisticated (broadcasting) functions +> * tools for integrating C/C++ and Fortran code +> * useful linear algebra, Fourier transform, and random number capabilities +> +> Besides its obvious scientific uses, NumPy can also be used as an efficient +> multi-dimensional container of generic data. Arbitrary data-types can be +> defined. This allows NumPy to seamlessly and speedily integrate with a wide +> variety of databases. + +To import libraries into your Python application, do the following: + +```python +# The %... is an iPython thing, and is not part of the Python language. +# In this case we're just telling the plotting library to draw things on +# the notebook, instead of on a separate window. +%matplotlib inline +# the line above prepares IPython notebook for working with matplotlib + +import numpy as np # imports a fast numerical programming library +import scipy as sp #imports stats functions, amongst other things +import matplotlib as mpl # this actually imports matplotlib +import matplotlib.cm as cm #allows us easy access to colormaps +import matplotlib.pyplot as plt #sets up plotting under plt +import pandas as pd #lets us handle data as dataframes +#sets up pandas table display +pd.set_option('display.width', 500) +pd.set_option('display.max_columns', 100) +pd.set_option('display.notebook_repr_html', True) +``` + +The way to understand these imports is as follows: _import the library `library` +with the alias `lib`_ where `library` could be `numpy` or `matplotlib` or +whatever you want and `lib` is the alias used to refer to that library in our +code. Using this flow, we can call methods like `plt.plot()` instead of +`matplotlib.pyplot.plot()`. It makes life easier. + + +**NOTE:** It is not necessary to import _all_ of these libraries all of the +time. You should only import the ones you really need. I listed a bunch above +to give you a sampling of what's available. + +**NOTE:** DO NOT include `%matplotlib inline` in your Python scripts unless +you're working in the Jupyter notebook. + + +At the end of this course, someone should be able to `import +your_kinetics_library` to use the kinetics library that you are about to start +writing. + + +## The Very Basics + + +We'll fly through this part because you should already know it. If you don't +understand something, please Google it and/or refer to the [Python +Tutorial](https://docs.python.org/3/tutorial/). I do not want to recreate the +Python tutorial here; instead, I'll just summarize a few important ideas from +Python. We'll give more details a little later on how some of these language +features work. + +Another very helpful resource that explains the basics below (and few additional +topics) can be found here: +[https://learnxinyminutes.com/docs/python/](https://learnxinyminutes.com/docs/python/). + + +### Calculating + + +We can tell the type of a number or variable by using the `type` function. + +```python +type(3), type(3.0) +``` + +Remember, every variable in python gets a type. Python is a strongly typed +language. It is also a dynamic language, in the sense that types are assigned at +run-time, rather then "compile" time, as in a language like C. This makes it +slower, as the way data is stored cannot be initially optimal, as when the +program starts, you dont know what that variable will point to. + + +All the usual calculations can be done in Python. + +```python +2.0 + 4.0 # Adding two floats +``` + +```python +2 + 4 # Adding two ints +``` + +```python +1.0 / 3.0 # Dividing two floats +``` + +```python +1 / 3 # Dividing two ints +``` + +Note that in Python 2, the division of two ints would not be interpreted as a +float; it is integer division. This is new in Python 3! Now, if you want +integer division you have to use the `//` operator. + +```python +1 // 3 # Integer division +``` + +```python +2**5 # Powers +``` + +```python +3 * 5 # Multiplication +``` + +#### More advanced operations + +We can use `numpy` to do some more advanced operations. + +```python +np.pi * np.exp(2.0) + np.tanh(1.0) - np.sqrt(100.0) +``` + +Notice that I am always writing my floats with a decimal point. You don't +really need to do that in Python because Python will automatically convert +between types. For example: + +```python +type(np.pi * np.exp(2.0) + np.tanh(1.0) - np.sqrt(100.0)), type(np.pi * np.exp(2) + np.tanh(1) - np.sqrt(100)) +``` + +However, I like to make the types as explicit as I can so there's no confusion. + + +### `print` + + +The `print` function is the basic way to write information out to the screen. I +will briefly review the new form of the `print` function. In Python 2, `print` +was a `statement` rather than a `function`. + +```python +print('Good morning! Today we are doing Python!') # Basic print +print(3.0) # Print a float +print('{} is a nice, trancendental number'.format(np.pi)) # Print just one number +print('{} is nice and so is {}'.format('Eric', 'Sarah')) # Print with two arguments +print('{0:20.16f}...: it goes on forever but {1} is just an int.'.format(np.pi, 3)) # Print with formatting in argument 0 +``` + +Here are some additional resources for the `print` function and formatting: +* [7. Input and Output](https://docs.python.org/3/tutorial/inputoutput.html) +* [Formatted Output](https://www.python-course.eu/python3_formatted_output.php) +* [`Print` function](https://docs.python.org/3/library/functions.html#print) + + +### Variables + + +We'll have more to say about variables in Python later. For now, here's how you +store them syntactically: + +```python +a = 1.0 +b = -1.0 +c = -1.0 +x = (1.0 + np.sqrt(5.0)) / 2.0 +val = a * x**2.0 + b * x + c +print('{0}x^2 + {1}x + {2} = {3}'.format(a, b, c, val)) +``` + +Python has this nice feature where you can assign more than one variable all on +one line. It's called the multiple assignment statement. + +```python +a, b, c = 1.0, -1.0, -1.0 +x = (1.0 + np.sqrt(5.0)) / 2.0 +val = a * x**2.0 + b * x + c +print('{0}x^2 + {1}x + {2} = {3}'.format(a, b, c, val)) +``` + +Looks a little cleaner now. + + +### Lists and `for` loops + + +Lists are central to Python. Many things behave like lists. For now, we'll +just look at how to create them and do basic operations with them. I will not +go through all the details. Please refer to +[Lists](https://docs.python.org/3/tutorial/introduction.html#lists) for +additional examples. + +```python +primes = [2, 3, 5, 7, 11, 13] # A list of primes +more_primes = primes + [17, 19] # List concatentation +print('First few primes are: {primes}'.format(primes=primes)) +print('Here are the primes up to the number 20: {}'.format(more_primes)) +``` + +Notice that Python knows that type of `primes`. + +```python +print('primes is of type {}'.format(type(primes))) +``` + +The `len` function can provide the number of elements in the list. + +```python +print('There are {} prime numbers less than or equal to 20.'.format(len(primes))) +``` + +Now that we know what a list is, we can discuss `for` loops in Python. The +`for` loop iterates over an iterator such as a list. For example: + +```python +for p in more_primes: + print(p) +``` + +A useful iterator (but not a list!) is the `range` function. + +```python +print(range(10)) +print(type(range(10))) +``` + +It's not a list anymore (it used to be in Python 2) and therefore can't be +sliced like a list can (see below). Still, you can use it in `for` loops which +is where it finds most of its use. + +```python +for n in range(10): + print(n) +``` + +There is something called a _list comprehension_ in Python. List comprehensions +are just a way to transform one list into another list. + +```python +not_all_primes = [p // 3 for p in more_primes] +print('The new list is {}'.format(not_all_primes)) +``` + +We can also count the number of each element in the list. There are a number of +ways of doing this, but one convenient way is to use the `collections` library. + +```python +import collections +how_many = collections.Counter(not_all_primes) +print(how_many) +print(type(how_many)) +``` + +We see that there are 2 ones, 1 two, 1 three, etc. + +We can even find the most common element of the list and how many occurrences of +it there are and return the result as a list. + +```python +how_many_list = how_many.most_common() +print(how_many_list) +print(type(how_many_list)) +``` + +We see that the result is a list of tuples with the most common element of our +original list (`not_all_primes`) displayed first. We want the most common +element of our original list, so we just access the first element using a simple +index. + +```python +most_common = how_many_list[0] +print(most_common) +print(type(most_common)) +``` + +We're almost there. We recall the first element of this tuple is the value from +our original list and the second element in the tuple is its frequency. We're +finally ready to get our result! + +```python +print('The number {} is the most common value in our list.'.format(most_common[0])) +print('It occurs {} times.'.format(most_common[1])) +``` + +List indexing is also very important. It can also do much more than what we did +above. + +```python +print(primes[2]) # print the 3rd entry +print(primes[2:5]) # print the 3rd to 5th entries +print(primes[-1]) # print the last entry +print(primes[-3:]) # print the three entries +``` + +Other types of slices and indexing can be done as well. I leave it to you to +look this up as you need it. It is a **very** useful thing to know. + + +Two convenient built-in functions are `enumerate` and `zip`. You may find +various uses for them. + +* `enumerate` gives a representation of a list of tuples with each tuple of the + form `(index, value)`. This provides an easy way to access the `index` of the + value in the `list`. +* `zip` takes elements from each list and puts them together into a + representation of a list of tuples. This provides a nice way to aggregate + lists. + + +We'll make two lists for the following examples: + +```python +species = ['H2', 'O2', 'OH', 'H2O', 'H2O2'] +species_names = ['Hydrogen', 'Oxygen', 'Hydroxyl', 'Water', 'Hydrogen Peroxide'] +``` + +#### `enumerate` example + +```python +print(enumerate(species)) +``` + +Notice that `enumerate()` just returns an iterator object. To actually see +what's in the iterator object, we need to convert the iterator object to a list + +```python +print(list(enumerate(species))) +``` + +We see that we have a list of tuples (in the form `(index, value)` where `index` +starts from 0). Here's just one way that this might be used: + +```python +for i, s in enumerate(species): + print('{species} is species {ind}'.format(species=s, ind=i+1)) +``` + +What happened is that the `for` loop iterated over the iterable (here +`enumerate`). The first index in the `for` loop corresponds to the first entry +in the `enumerate` tuple and the second index in the `for` loop corresponds to +the second entry in the `enumerate` tuple. + + +#### `zip` example + + +Let's see how `zip` works. We'll aggregate the `species` and `species_names` +lists. + +```python +print(zip(species, species_names)) +print(list(zip(species, species_names))) +``` + +```python +for s, name in zip(species, species_names): + print('{specie} is called {name}'.format(specie=s, name=name)) +``` + +We see that this worked in a similar way to `enumerate`. + + +Finally, you will sometimes see `enumerate` and `zip` used together. + +```python +for n, (s, name) in enumerate(zip(species, species_names), 1): + print('Species {ind} is {specie} and it is called {name}.'.format(ind=n, specie=s, name=name)) +``` + +### Opening Files + + +There are a variety of ways to open files in Python. We'll see a bunch as the +semester progresses. Today, we'll focus on opening and reading text files. + +```python +species_file = open("species.txt") # Open the file +species_text = species_file.read() # Read the lines of the file +species_tokens = species_text.split() # Split the string and separate based on white spaces +species_file.close() # Close the file! +``` + +```python +print(species_tokens) +print(type(species_tokens)) +``` + +Notice that we get a list of strings. + + +Here's a better way to open a file. The `close` operation is handled +automatically for us. + +```python +with open('species.txt') as species_file: + species_text = species_file.read() + species_tokens = species_text.split() +``` + +### Dictionaries + + +Dictionaries are extremely important in Python. For particular details on +dictionaries refer to +[Dictionaries](https://docs.python.org/3/tutorial/datastructures.html#dictionaries). +From that tutorial we have a few comments on dictionaries: + +> Unlike sequences, which are indexed by a range of numbers, dictionaries are +> indexed by keys, which can be any immutable type; strings and numbers can +> always be keys. +> +> It is best to think of a dictionary as an unordered set of key: value pairs, +> with the requirement that the keys are unique (within one dictionary). A pair +> of braces creates an empty dictionary: {}. Placing a comma-separated list of +> key:value pairs within the braces adds initial key:value pairs to the +> dictionary; this is also the way dictionaries are written on output. +> +> The main operations on a dictionary are storing a value with some key and +> extracting the value given the key. + + +Let's create a chemical species dictionary. + +```python +species_dict = {'H2':'Hydrogen', 'O2':'Oxygen', 'OH':'Hydroxyl', 'H2O':'Water', 'H2O2':'Hydrogen Peroxide'} +print(species_dict) +``` + +The entries to the left of the colon are the keys and the entries to the right +of the colon are the values. To access a value we just reference the key. + +```python +print(species_dict['H2']) +``` + +Pretty cool! + +Suppose we want to add another species to our dictionary. No problem! + +```python +species_dict['H'] = 'Atomic Hydrogen' +print(species_dict) +print(species_dict['H']) +``` + +Why should we use dictionaries at all? Clearly they're very convenient. But +they're also fast. See [indexnext |previous |How to Think Like a Computer +Scientist: Learning with Python 3: 20. +Dictionaries](http://openbookproject.net/thinkcs/python/english3e/dictionaries.html) +for a decent explanation. diff --git a/lecture06/notebooks/python_2.ipynb b/lecture06/notebooks/python_2.ipynb @@ -0,0 +1,893 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Python\n", + "\n", + "* Booleans and Control Flow\n", + "* Functions\n", + "* Exceptions\n", + "* Plotting" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll be embedding some HTML into our notebook. To do so, we need to import a\n", + "library:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from IPython.display import HTML" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll also probably use `numpy` so we might as well import it too:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Booleans and Control Flow\n", + "\n", + "These are pretty standard things in any language. I'll just show you a little\n", + "bit of syntax here. However, I will put a slight emphasis on the exception\n", + "testing. It's important." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Testing for membership" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "True\n" + ] + } + ], + "source": [ + "some_primes = [2, 3, 5, 7, 13]\n", + "print(4 in some_primes)\n", + "print(13 in some_primes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Some basic if/elif/else statements" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "x is definitely not prime.\n" + ] + } + ], + "source": [ + "x = 9\n", + "if x in some_primes:\n", + " print('x is prime!')\n", + "elif x > 13:\n", + " print('x may or may not be prime.')\n", + "else:\n", + " print('x is definitely not prime.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Breaking out of a loop:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Only even prime.\n" + ] + } + ], + "source": [ + "for x in some_primes:\n", + " if x == 2:\n", + " print('Only even prime.')\n", + " break\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Continuing a loop. Notice that everything after the continue statement is\n", + "ignored." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "2\n", + "3\n", + "4\n", + "5\n", + "Done with this.\n" + ] + } + ], + "source": [ + "i = 0\n", + "while i < 10:\n", + " i += 1\n", + " if i <= 5:\n", + " print(i)\n", + " continue\n", + " print(i-1)\n", + " else:\n", + " print('Done with this.')\n", + " break\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Some basic exception handling\n", + "\n", + "Python has a number of built-in exceptions ([Built-in\n", + "exceptions](https://docs.python.org/3/library/exceptions.html), [Python Standard\n", + "Exceptions](https://www.tutorialspoint.com/python/standard_exceptions.htm)). It\n", + "is usually a good idea to let Python raise exceptions for you since it's really\n", + "good at it. However, there are times when you may want to write your own\n", + "exception (we won't talk about that now) or when you want to press ahead even in\n", + "the face of an error.\n", + "\n", + "I can make that last statement clearer. You have two options: catch and\n", + "respond to errors when they're raised or ignore them. If you ignore them, then\n", + "Python's default exception-handling behavior takes over which will ultimately\n", + "print out the error message and terminate the program. If you respond to the\n", + "errors, then you need to tell your program what to do. In essence, you will\n", + "shift the control flow of your program if you choose this second option.\n", + "\n", + "Let's look at an example or two." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "ename": "ZeroDivisionError", + "evalue": "float division by zero", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mZeroDivisionError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-7-3b7789856ab2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mZeroDivisionError\u001b[0m: float division by zero" + ] + } + ], + "source": [ + "x, y = 1.0, 0.0\n", + "z = x / y\n", + "z**2.0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Python took care of error for us and terminated the program at the second line.\n", + "\n", + "But perhaps a division by zero isn't the end of the world. What if we have a\n", + "piece-wise function?" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "z = 0.0\n" + ] + } + ], + "source": [ + "x, y = 1.0, 0.0\n", + "try:\n", + " z = x / y\n", + "except ZeroDivisionError:\n", + " z = 0.0\n", + "print('z = {}'.format(z))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This could, of course, have been handled with an `if-else` block.\n", + "\n", + "One old motivation for using exception handling is to check the input arguments\n", + "of a function for validity. You can still do this, but the latest advice is to\n", + "just let Python's exception handler deal with it and terminate the program if\n", + "need be." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Functions\n", + "\n", + "#### Python functions are first class:\n", + "\n", + "* You can assign variables to them\n", + "* You can pass them into functions\n", + "* You can return them from functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example: " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original maximum gives 13 and new maximum gives 13.\n" + ] + } + ], + "source": [ + "alist = [1,13,5,7]\n", + "newmax = max # Assign newmax to the function max\n", + "mymax1 = max(alist) # Get the maximum of the list using the max built-in\n", + "mymax2 = newmax(alist) # Get the maximum of the list using the new max function\n", + "print('Original maximum gives {0} and new maximum gives {1}.'.format(mymax1, mymax2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The syntax for defining functions is pretty straightforward." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def rect(w,h):\n", + " return w * h\n", + "def circle(r):\n", + " return np.pi * r * r\n", + "def parabola(x, a, b, c):\n", + " return a * x * x + b * x + c" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that the function name is preceded by the keyword `def`. The end of the\n", + "line **must** have a colon! The function body is indented. The quantity to be\n", + "returned is returned using the `return` statement.\n", + "\n", + "Because functions are first class, we can pass functions to other functions." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parab_extrema(a, b, c, parab):\n", + " x_extreme = - b / 2.0 / a # Location of max or min\n", + " x_left = x_extreme - 1.0 # Point to the left of max or min\n", + " x_right = x_extreme + 1.0 # Point to the right of max or min\n", + " p_left = parab(x_left, a, b, c) # Value at left point\n", + " p_right = parab(x_right, a, b, c) # Value at right point\n", + " p_extreme = parab(x_extreme, a, b, c) # Value at max or min\n", + " # Check if extremum is maximum or minimum and print out result\n", + " if (p_left > p_extreme) & (p_right > p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme))\n", + " elif (p_left < p_extreme) & (p_right < p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme))\n", + " else:\n", + " print('Something went wrong.')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The extremum for this parabola has coordinates (-2.500, -4.250) and is a minimum.\n", + "The extremum for this parabola has coordinates (-0.500, 6.250) and is a maximum.\n" + ] + } + ], + "source": [ + "a, b, c = 0.2, 1.0, -3.0\n", + "parab_extrema(a, b, c, parabola)\n", + "a, b, c = -5.0, -5.0, 5.0\n", + "parab_extrema(a, b, c, parabola)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are some other convenient ways to interact with function arguments.\n", + "\n", + "One thing you may wish to do is provide a default argument to the function. If\n", + "that argument is not specified when the function is called, then the default\n", + "value is assumed. This is a type of keyword argument.\n", + "\n", + "Let's return to our nice parabola example. We'll make $b=0$ the default." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parabola(x, a, c, b=0.0):\n", + " return a * x * x + b * x + c" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that we had to move our default argument to a position _after_ the\n", + "mandatory arguments. That hurts the readability a little bit in this example,\n", + "but we'll press forward regardless.\n", + "\n", + "Now call the `parab_extreme() function` again." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parab_extrema(a, b, c, parab):\n", + " x_extreme = - b / 2.0 / a # Location of max or min\n", + " x_left = x_extreme - 1.0 # Point to the left of max or min\n", + " x_right = x_extreme + 1.0 # Point to the right of max or min\n", + " p_left = parab(x_left, a, c) # Value at left point\n", + " p_right = parab(x_right, a, c) # Value at right point\n", + " p_extreme = parab(x_extreme, a, c) # Value at max or min\n", + " # Check if extremum is maximum or minimum and print out result\n", + " if (p_left > p_extreme) & (p_right > p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme))\n", + " elif (p_left < p_extreme) & (p_right < p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme))\n", + " else:\n", + " print('Something went wrong.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We changed the\n", + "[API](https://en.wikipedia.org/wiki/Application_programming_interface) a little\n", + "bit and so we had to update the calls to `parab()`. However, everything works\n", + "just fine if we're careful." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It's probably better to give all the parameter arguments default values. Let's\n", + "re-write the API again." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The extremum for this parabola has coordinates (0.500, -1.250) and is a minimum.\n" + ] + } + ], + "source": [ + "def parabola(x, a=1.0, b=-1.0, c=-1.0):\n", + " return a * x * x + b * x + c\n", + "\n", + "def parab_extrema(parab, a=1.0, b=-1.0, c=-1.0):\n", + " x_extreme = - b / 2.0 / a # Location of max or min\n", + " x_left = x_extreme - 1.0 # Point to the left of max or min\n", + " x_right = x_extreme + 1.0 # Point to the right of max or min\n", + " p_left = parab(x_left, a, b, c) # Value at left point\n", + " p_right = parab(x_right, a, b, c) # Value at right point\n", + " p_extreme = parab(x_extreme, a, b, c) # Value at max or min\n", + " # Check if extremum is maximum or minimum and print out result\n", + " if (p_left > p_extreme) & (p_right > p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme))\n", + " elif (p_left < p_extreme) & (p_right < p_extreme):\n", + " print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme))\n", + " else:\n", + " print('Something went wrong.')\n", + "\n", + "parab_extrema(parabola)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great! Looks pretty nice.\n", + "\n", + "But there's more! We can also provide _positional_ and _keyword_ arguments to a\n", + "function. This allows permits a variable number of arguments to be passed to a\n", + "function.\n", + "\n", + "* positional arguments: `def func(*args):`\n", + " + Python collects all the remaining positional arguments into a tuple. You\n", + " can then access the tuple with the usual indexing.\n", + "* keyword arguments: `def func(**kwargs):`\n", + " + Python collects all the remaining keyword arguments into a dictionary. You\n", + " can then access the dictionary with the usual indexing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Variable Positional Arguments\n", + "\n", + "We will once again work with the quadratic equation example. This time, we'll\n", + "just work with the `parabola` function to save some space. Let's change the\n", + "`parabola` function to permit a variable number of arguments." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-1.0" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def parabola(x, *args):\n", + " return args[0] * x * x + args[1] * x + args[2]\n", + "parabola(1.0, 1.0, -1.0, -1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Seems to work okay. But this is not a very robust code. Everything breaks if\n", + "we don't provide the exact number of necessary arguments." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "tuple index out of range", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-17-4a52e1254eed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparabola\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m<ipython-input-16-f85ec532a54e>\u001b[0m in \u001b[0;36mparabola\u001b[0;34m(x, *args)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mparabola\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0mparabola\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIndexError\u001b[0m: tuple index out of range" + ] + } + ], + "source": [ + "parabola(1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Variable keyword arguments\n", + "\n", + "We can make our API more flexible. Let's give more descriptive names to the\n", + "coefficients $a$, $b$, and $c$. We'll call $a$ the `width` since it controls\n", + "the width of the parabola, $b$ `trans` since it controls the horizontal\n", + "translation of the parabola, and we'll call $c$ `shift` since it controls the\n", + "vertical shift of the parabola. Our `parabola` function might now look like:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parabola(x, **kwargs):\n", + " print(kwargs)\n", + " return kwargs['width'] * x * x + kwargs['trans'] * x + kwargs['shift']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Calling it gives:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'width': 1.0, 'trans': -1.0, 'shift': -1.0}\n" + ] + }, + { + "data": { + "text/plain": [ + "-1.0" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "parabola(1.0, width=1.0, trans=-1.0, shift=-1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note:** Using variable positional and keyword arguments provides exceptional\n", + "flexibility in how you design your programs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One final note about variable arguments: You can perform the reverse operation\n", + "by passing in the `*` or `**` operators to the function. This will _unpack_ the\n", + "arguments whereas the previous pattern _packed_ the arguments. Let's take a\n", + "quick look." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def parabola(x, a, b, c):\n", + " return a * x * x + b * x + c" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-1.0" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Store coefficients in a list\n", + "coeffs = [1.0, -1.0, -1.0]\n", + "parabola(1.0, *coeffs)\n", + "\n", + "# Store coefficients in a dictionary\n", + "coeffs = {'a':1.0, 'b':-1.0, 'c':-1.0}\n", + "parabola(1.0, **coeffs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "---\n", + "\n", + "## Plotting\n", + "\n", + "There are many, many ways to make plots in Python. The most common way is to\n", + "use [`matplotlib`](https://matplotlib.org/).\n", + "\n", + "Another package, which is gaining popularity, is called\n", + "[`seaborn`](https://seaborn.pydata.org/).\n", + "\n", + "I don't care which package you use, as long as your plots are readable and\n", + "reproducible." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make plots in the Jupyter notebook, you need to include the line `%matplotlib\n", + "inline` before you make any plots. This line ensures that the plots will be\n", + "displayed in your notebook and not in a separate window." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, you should `import matplotlib`:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import matplotlib # Import all of matplotlib\n", + "import matplotlib.pyplot as plt # Only import pyplot (which includes the plot function) and give it the alias `plt`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now you're basically ready to do some plots.\n", + "\n", + "**WARNING!** When making plots in an actual Python script, you must **always**\n", + "include the command `plt.show()` at the **end** of your program. **Always.** If\n", + "you don't do so, then your plots will not display and you will be wondering\n", + "where they are. However, when plotting in the Jupyter notebook, there is no\n", + "need to use `plt.show()`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can generate some toy data using `numpy`." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "x = np.linspace(-2.0*np.pi, 2.0*np.pi, 500) # x-grid\n", + "ys = np.sin(x) # sin function\n", + "yc = np.cos(x) # cos function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now plot!" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "<matplotlib.text.Text at 0x10fb8c080>" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaIAAAEtCAYAAABK7WRiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd4HNXVuN+j3mwVW5aLrObebTC2MbaxMT2AgYQEQktC\nQgjpIYV835f8SEJ6QiqB0AIkBEIglNCxaS64494ty71IlmTJ6uX+/phZ78zdVd/dmZXmfZ59tPfO\n3J2z0mjOveece44opfDw8PDw8HCKGKcF8PDw8PDo23iKyMPDw8PDUTxF5OHh4eHhKJ4i8vDw8PBw\nFE8ReXh4eHg4iqeIPDw8PDwcxVNEHo4jIp8RESUi7zktS7QjIo+bv8t7nJYlnIjIPeb3fNxpWTx6\nTpzTAnj0HkQkDrgJuB6YAgwAaoBjQDGwFHhHKbXaMSE9OkREBFgEXAOcC+QASUA5sBl4C/i7Uuq4\nY0J69Co8ReQREkQkG3gNmG7prgcEGAOMBS4HTgEZ2vBTwE7gQPgl9WgPERkNPANMs3Q3YkwocoDB\nwEXAj0Xkf5RSv4+8lACUYdwzRx26vkcI8UxzHqHiHxhKqBr4LjBEKZWslMoA0jEeXn8BKvWBSqkX\nlFJjlVK3RFJgDzsiMgVYiaGEKoC7gRFKqUSlVBbGqmgB8DcgAbjaKVmVUn8275nvOyWDR+jwVkQe\nPUZExgIXm83PKaWesx5XSlUDi4HFInJXpOXz6BgRSQWeAzIxzKgLlVIl1nOUUo3Ae8B7IvIH4CsR\nFtOjl+KtiDxCwSTL+1faO1EpVa/3tResICIl5rH5IpIlIveJyD4RaRCRwyLysIgMae+aIlIgIn8S\nkZ0iUisi1SKyTkS+Zz6Au4yIzBORP4jIKhE5IiKNInJCRN4QkU+0M+5MMIGIxIrIN0RkoylXuYi8\nIiLT2xpvfsZMEfmvef5pEdkgIl8XkZ78P98BjARaget1JaSjlNoI3N6GfNeav4dS8+90SESeEpGz\n2vlOg0Tk1yKyRURqRKReRA6KyAoR+bGI5GvntxmsYPYr8++eZ94jh0xZ9onIb0Skf3vfT0Qmishj\n5vn1IlIpIstF5A4RiW9vrEc3UEp5L+/VoxdwHaDM14hujP+MOfa9IMdKzGM3Wd7XYPiffNfcB2S2\n8dnXAnWWc2swfB6+9iYgp4vyplnGK6AKw89l7ftrG2MfN4/fC7xhvm/EMGn6xtYB57Yx/nqg2XJu\nBdBkvn/O8vn3dPE77TLHvdaD+yAGeMIiW7Mpn6/dAnwpyLh84Ig2rhxDKfr67tDG3GP2Px7k83xj\nFgEnLX+jJsuxNUB8G9/jK6asvnOrtd/5u0CK0/93venlrYg8QsE6y/v7zcCFUPMnjIfabKVUKoYy\nWIThcyoAAnwFInIOhuM9DvgpkGuOTQZmA2sxVnNPdlGWVoyH/jXAAKVUf6VUOoZZ6yvAaeB2Ebmu\nnc/4MnAO8CkgTSnVDyPScAuGL+YPQb7PCAz/TCxG5NoIpVQmhg/uLgyfTZf9NiIyDBhlNl/u6ngL\n3wVuwXhY/wBjcpAJ5AL/xlBUfxaRedq4/wcMAfYA84AEZfikkjH+PvdiRF52lceBDcAkpVR/jHvm\nNqABw5/5BX2AiFyNca/VmN8n2/zbpACXAruB+cDvuiGPR1s4rQm9V+94YZ8JN2D4hO7FUBbZHYz9\nDB2viI5hPPT143eZx4uDHFtmHvtiG9fNwj8Tnx7C38XN5me+G+TY45bf05wgx8+2HM/Tjj1q9u8A\nkoKM/T/L2Hu6IO+FlnFBV2Kd+Iw0/KvCnwc5HosRvq+AD7Rj28z+T3XhevfQ8YpoC5AY5PifzOPv\nBJHRd79d0sZ1R2AoqSaMgBxH/+96y8tbEXmEii8A92GYmRKAhcD/Ai8CJ0RktYjcaO5R6Q4PKaVO\nBul/0fxZaPX3mKuH8zBWTI8G+0ClVDnwutm8qJtyBeO/5s9ZIhLbxjlLlVLLgsi0DjhkNif6+s3f\n27Vm83cqiK8N+D1Q2w15B1jel3djPBi/v/4Yf/9f6QeVUi3AT8zmXBEZbDlcZf5s19fXDe5TSjUE\n6ffdMxO1/vkYZsItSqk3g32gUmovRmRhnHm+RwjwFJFHSFBKNSql7gKGYzi+n8YwY/gqL56DEeL9\nr2461de00X/Y8t66P2m2+TMNOCQix4K9MExjmHJ3GhGJE5HbTKf8UdMRrkTE57cBw8SW2cXvY/1O\n1rFF+L/f+8EGKaVOYzeTRhJfIMJGpVRFG+d8gOF7sZ4Pxv4zgF+KyP0iskBEkkMgU0f3jP638d0z\no9q6X8x7xndel+4Zj7bxwrc9QopS6gTwV/OFiOQAVwI/xPjHvQ5YThAfSAdUt3G9essiyxrN5Jtd\nx2FsxOyIlM4KIiJpwJv4H0hgBBiUYviPsFwzFWPzpU7Q72PiW+1Yv4/V73aknbGH2znWFtaVZlY3\nxoNfvjavb/6tyjB+N9bv80sMk+RVwJ3mq1lE1gAvAA8rpQL2n3WCtn7Hvt+v/vzz3TOJhPie8Wgf\nb0XkEVaUUseVUo9gzIB9KWE+F4FL++7tjUop6cTrM1347B9gKKEy4FaMqLsUpdQgpdRgYJjl3O6a\nIiPJdsv7KT38rKSuDlBKNSilFmGkE/oVhulLWdq7xNhsG25898xLnbxn7omATH0CTxF5RASlVBnw\nktkcHYFL+pReOMwnvmi4ryqlnjRXgVY6M5vuKqWW90PbOa+9Y0FRSh3GMKOCsSrpDj758to6QUSS\n8PujSvXjSqmVSqnvKaXOxTCb3YCR9ikbeKSbcnUF3z3T5nfwCA+eIvKIJDXmz8YIXOtD82eWiMwM\n8Wfnmj8/auP4hSG+HhjZDnzmKT38GTiTHaHdzbDt8JD58xIz7L1DNF/fevPnKDMcPBjz8JvD1rdx\nDgBKqRql1DP4N82e3d3Nx13Ad89Mbuc7eIQBTxF59BgRKTSj1No7JwX/HpcN4ZZJKbUDw8QD8Kv2\ndsOLSIqIJHbh40+ZPyfpB0z/0f924bM6hVJKAc+bzW+0Ie/X6L7f4kEMZRcDPCMiBe2dbJrK/mrp\negsj+i0e+E6Q82MxTJpgRAwesxxLaOdSdb7TMKIxw8kS4CBGGPev2ztRRNoKQvHoBp4i8ggFE4Cd\nIvIfEfmkWFLuiEiqiFyJsYek0OzuaqBCd/kaxp6mecASEZnjm8WLkV5nqoj8COMB3JXQ4bfNn/eJ\nyPm+kHRzJbEEezh0KPk5hqN9HPCiiBSa100WkW9ghEefamd8m5gRd5/AWHUVAb4USL6/GSKSYH7f\nxzCi80ZZxtcAPzObXxOR/zWVsm/D7NPAHIxgjv/TLr9FRH4mIuf4lJIYzMDY8wOwpp1ovJCglGrC\n2JCsgBtE5EURmeo7bn7/WSLyW4xsHh6hwumNTN4r+l/AJdjT2yiM/SyVWl8z8D9Bxn+Gjje0zm/n\n+r7PLwhy7DJNjnqMIIMmTbb8LnzfIgwfhzUlz2nL9764LZnoRAoejMSiCvhMkGMdpfh5oqPP7+C7\njcVYsVp/N/UEptypBu7UxsYSmOLHOq5FH2OOq9TGnMSehqkUmKyNuYeON7QG3A/m8QLfOW0c/yzG\nBMZ6L5/Ufu9Bx3qv7r28FZFHj1HG5r8xwLcxNgvuMQ+lYTxk1mNstpyilPpZ0A8Jn2yvYwRH3GvK\n0YCxH+cUsAL4BXC2Ump/Fz6zGJiBsS/qBMYDuBJ4CjhHKfVWKL+Ddu1nMDbqvmpeMwEjM8E3gE/i\n37fV3c/fgVEG4lqM77cXQ9GlYjjz3wK+BRQqpf6ijW1RSt2KsbJ6y5QvDaNm0NPADH2MySKM1d5y\njND0NAxFtAnj7zNBKbWpJ9+rKyil/oZxP/8e2IqhQPtjKKP3MFISjYmUPH0BMWcAHh4eHh4ejuCt\niDw8PDw8HMVTRB4eHh4ejuIpIg8PDw8PR/EUkYeHh4eHo3hJTzvBwIEDVUFBgdNieHh4eEQV69at\nK1NKdVgo01NEnaCgoIC1a9c6LYaHh4dHVCEindoW4ZnmPDw8PDwcxVNEHh4eHh6O4ikiDw8PDw9H\n8RSRh4eHh4ejeIrIw8PDw8NRolIRichjInJCRLa0cVxE5I8iskdENonIWZZjl4rITvPY3ZGT2sPD\nw8MjGNEavv048GfgyTaOX4ZRK2UUMBN4AJhpFue6H7gIOASsEZGXlVLbwibpB7+Gk8WQfy6MvQJS\nssJ2qfZobVUs21PGsj1l7D1xmobmVnL6JzEtL4OPTRpCZmq4a455hASl4NAa2P0WHN8GjachNRuG\nToMJV0N6bsefERaxFJsPn2Lx9hPsPFZFdX0zA9ISmTSsP5dOGELegO7W6/PoC0Rt9m2zguQrSqmJ\nQY79FaO2zdNmeycwH6MOyT1KqUvM/u8DKKV+3t61pk+frrq9j+ixy+DACuN9fArMuB3mfQcS07r3\ned3g7W3H+fnr2ykurQl6PCE2hpvPzedrC0eRntxmIVMPpzmwEt78Hzi8ro0TBKZcDwv/H/TvSp2/\nnrHhYCU/e3U7q0vK2zznyilD+d6lY8jN9BRSX0JE1imlOixfH5WmuU4wDKPkr49DZl9b/QGIyO0i\nslZE1paWlnZfkuqj/vdNtbD89/DXeXBsc/c/s5PUN7Vw17Mb+cKTa9tUQgCNLa08umwfl/9hKZsO\nVYZdLo8u0toKi38Ej13ajhICULDxafjwzxERSynFH5fs5uMPrGhXCQH8d+MRLvvDUl7ffLTd8zwc\norXV0cv3VkXUY5RSDymlpiulpmdnd5ihoq0Pgepjgf3le42V0r4PeiZkO1TVN3HTI6t4fv2hTo85\nXFnHdQ9+yHs7T4RNLo8u0twIz30Wlt1Hp2reDZ0GC38YfrFaWvnOc5u47+1dtLR2zqpSXd/Ml55a\nz2PLvCrbrkEpePfn8J/PO6qMotVH1BGHgeGWdq7ZF99Gf3hQCj75pDGLXfc3OH3cf6yxGp76JNz6\nMgyfEdLL1jY287m/rWHt/gpbf2yMcMXkISwcl0NaYixbDlfxzOoDHDlVf+achuZWbn9yHY995hzm\njBoYUrk8ukhrC7zwRdj2YuCx0ZfC+EWQOghKd8BHfzcmPdc9AXGJYRVLKcX3/7OZ59YFTnLmjhrI\nlZOHMqh/IvvKavjXmoPsOFZtO+fHr2wjPi6Gm2flh1VOj07wwW/g/V8Y75Oz4PJfg0jExeitPqKP\nAV8BLscIVvijUmqGiMQBu4CFGApoDfBppdTW9q7VIx+Rj8Yaw76/7nF7f3Im3P4eZBb07PNNlFJ8\n9emPeGWT3QRSlJ3Kn26YxoSh6bb+usYWfvnGDh5fUWLr75cUxwt3zmbkoH4hkcujGyz+kbkSspCW\nA9c+BEXz7f0tzVBRAgNHhl2sPy7ZzX1v77L1ZaUm8NvrprBg7CBbf2ur4vEVJfz89e00tfifNTEC\nD98ynYXjcsIur0cbbPwXvHC7ve+C/zN82CGiV/uIRORp4ENgjIgcEpHbROQOEbnDPOU1oBjYAzwM\n3AmglGrGUFBvAtuBZztSQiEjIRWu/ANc+CN7f10FPPc5wwQTAh5fURKghM7Ky+CFL50XoIQAkhNi\nueeqCfxk0QRbf3V9M3f8Yz31TS0hkcuji5QXw/I/2PuyiuDziwOVEEBsXESU0Io9ZfxusV0JDctI\n5vkvzQ5QQgAxMcLn5hTy6K3nkBwfe6a/VcFd/97I0VN1YZfZIwilu+CVb9j7ktKh8HxHxIlKRaSU\nukEpNUQpFa+UylVKPaqUelAp9aB5XCmlvqyUGqGUmqSUWmsZ+5pSarR57KcRF37ON+D879n7Dq+D\nd37S44/edKiSn7663dY3Jqcff/vsDNJT2o+Gu/ncAv7n8rG2vj0nTvOL13f0WC6PbpBVBDf/xwjN\nBkgbDLe8BBl5Xf+s2nJY9nvDVNwDyk438PV/bbB9TFZqAn+/bQaFA1PbHTtvdDb33ziN2Bi/2aey\ntomvP7Oh0z4mjxDRVG/4HZtq/X2xiXDDv0LuJugsUamIop7534cxl9v7VvzRCM/tJk0trXzv+c00\nW/6p+yXG8eDNZ3c6JPsLc4u49ix7EOHjK0p4f1cPogY9uk/RfPjiUiiYC9c93j0ltO8DeOA8WPz/\nYO2jPRLn/728ldLqhjNtEfjj9dMoyu7cVoQLxubwnUvG2PpW7yvnkaXFPZLLo4u8/ws4ruUC+Nhv\njL2ODuEpIicQgUX3Q38tcvzVuwxbfzd4bNk+th+tsvX9+rrJHc5U7WIJP140kbws+16PH760xTPR\nOUX/IXDrf7v3kFj3ODxxFVQfMdqLfwynuzep+GBXKa9qJt+vLBjZ5YCW2+cWMVcb84cluz0TXaQo\n3Qkr/mTvm/hxmHazM/KYeIrIKVKy4JoH7X3Ht8C+97r8UYcqagPs9oumDuXSiV3f1JiWGMfvPjUV\niwWF/SdredQLuXWO7kYxFS2ABMtqpeEULPlR2+e3QX1TCz98yT6DnjQsna8vHNXlz4qJEX77ySm2\nVXptYwv3vrK9nVEeIUEpY7Lbapns9hsKV/zOkUg5K54icpLCeTD5U8b7nInwuTdh5IVd/pj73t5F\nfZN/D0B6cjw/uGJ8t8U6Oz8zILT2z+/s8Wat0UZmPszX0il+9I8ONsUG8viKEkpO+v0JInDv1ROJ\ni+3e42NQv6QAE92rm4+yYm9Ztz7Po5NsfxlKltr7Lv2ZEaTgMJ4icpqL74VLfwm3vw95s7o8fMex\nKl74yL4V6u7LxjIwrWd7Sb550WgyLQEOdU0t/GHx7h59pkc71JTB+7+GhuqOz+0KM78IA60PfWWE\nhXeSqvomHnhvr63vxpl5TBme0SOxbpiRx6Rh9gfgb97cSbRuJ3E9rS3wjhabVbQAxl/tjDwaniJy\nmrRBMOsOI/y2G/z6jZ22KKbROWl8cvrwtgd0koyUBO662D5rfW7dIQ6W17YxwqNHLP0tvHsv/GEq\nrHwQmhs6HtMZYuPhsl/a+/a9DyXLOjX8kaX7OFXXdKbdLzGOuy4a086ITooVI9xzlX3LwPoDlby3\n0wuMCQubnoWynf62xBj3hcMmOR+eIopiNh6sZMkOezqe71wy1hYi2xOuP2c4BZasyc2tij+9462K\nQk7VUVjziPG+tgze+J6RKSFUjFgQuD/k3Z91GM5dXtPIo1pE2xfmFYUsU/vZ+Zks1PYe/eYtb1UU\nclqa4D0tr/Pk6yG75xOKUOEpoijmwfftJpOz8zO5cFzgpsLuEhcbw9c0h/Tz6w+z/2TbCVQ9usHK\nv0CLZUNz/1yYelNor7Hgf+zt/csD/QUaT6wooabRHy2ZlZrA5+YUhlSsb1082tbeeqSKd3Z4uQ5D\nyu63oHK/vx0TD/O/1/b5DuApIrdSV9Gu+WRv6Wne2GpPqPq1haOQEC+1r5oylCJLCHhLq/KSVoaS\nukpY+zd739xvQnxSaK+TNwtGLLT36WG8VrEaW3jywxJb3x3nF5GWGNr0lBOGpnP5pMG2voc+8PYV\nhZQxl8Otr8Coi432WbeELKVYqPAUkduoPACv3w33TYBnboSG00FPe/iDYptlZfyQ/swLQ5LSuNgY\nvnKBPXXMs2sPcaq2qY0RHl1i7aNGAlwfqdkw9cbwXOv879rbu98y9pUE4d/rDlJh+Rv3S4rj0zPD\nk6T0zvn2+2vVvnI2HvTKkYQMESicCzf+G+5cCfO+7bREAXiKyE20NMHDC2HVA9BUA/WVsD6wCO3J\n0w38Z709Uu5L80eEfDXk44rJQ8np74/Cq2tq4Z+rD4TlWn2KliZY9Vd738wvQnxyeK43fCYM0/JP\nfnh/oFitikeW2le9N83KD/lqyMfEYenMHjHA1vewl20hPAwaB/2HOi1FAJ4ichOx8XD2Z+x9qx8K\nqBPyzJqDNLb4+/KyUrhsot28EUoS4mK4dXaBre/xFftoanG2mFbUs+MVe2mQhDQ45/Phu54IzP6K\nvW/jM0bouIV3d5zggCU6MiE2hs9qf/9Q84V5Rbb2a5uPevvW+hCeInIbM26HWEtUUsU+KH73TLO5\npZV/rrKvRm45N7/bmws7y6dn5NmyJx+vauDtbcfbGeHRIWu03G9TrjfKgoSTsVdCuiVnXUsDbH3B\ndso/Vu23ta+aOpRB/UPss9KYPzqb0Tn+LBCtCp5ZfbCdER69CU8RuY207MBNZmsfO/N2yY4THK70\nzxST4mO47uye7xvqiIyUBD5+tj03nq4QPbpA6a7AqLXpnwv/dWPjYObtRuTUxI8bTmzLKuzAydqA\nJLe3nBv+AnYiEpDN45k1B2j2Vt3do6nOyLoeJXiKyI2cc5u9vfN1OGX4hP6x0j5bXTRlWIclHkLF\np2fYHxTL9pR5odzdZZ0WKTd8FuRMCH5uqDnrVrhrB3ziMcOJbfEtPrV6vy0IZnJuOpNze5ZFobNc\nPW0YKQn2Vffi7V4od7fY9C+4bxy8eCccWtvjEiDhxlNEbmT4TBhkyRWnWuCjv3Okso5le+z2/Jsj\nMFv1MX5of6ZqqV2e9swnXaexFjY8Ze/TJx/hJKk/pAZGWDY2t/LcWnv575vCFCkXjH5J8Syaal91\nP6WZCT06ydq/QXO9cZ89shBWPdjxGAeJSkUkIpeKyE4R2SMidwc5/h0R2WC+tohIi4hkmcdKRGSz\neayH9b/DhEigmWbDP3lh/aGA2erEYZFNWPjpmfaaOM+tO+gFLXSV7S9D/Sl/O2UAjF/knDwm7+8q\n5WSNf2Ntv6Q4rpwS2QirG7X7a9meMi9ooasc3wpHN1g6JLD+mcuIOkUkIrHA/cBlwHjgBhGxpZpW\nSv1aKTVVKTUV+D7wvlLKajBdYB7vsJa6Y0y6zqia6KNyPzvWvG075RNn50ZYKLhy8lD6JfnDeMtO\nN7J0t5cfrEtsfNrennojxPUsSW0o+M96+2roqilDSbaYyiLBxGHptmSoSsGLHx2JqAxRz8Zn7O2i\n+UYmdhcTdYoImAHsUUoVK6UagWeA9qaTNwBPt3PcnSRnwFj7LObc6rfOvE+IjeHKyZHfD5CcEMsV\n2nX1PU0eHTD3LkP5+GoFhWsDaxeorG1k+Xa7mfXasyI/0TGuazfP/Wf9IS//XGdpbTESnFqZcoMz\nsnSBaFREwwDrf8whsy8AEUkBLgWet3QrYLGIrBOR29u6iIjcLiJrRWRtaalDM37tBroidhWJGKaT\nheMGhSz5ZFe5Zpr91/32tuNU13uZFjpN4Ty4+i/w7d3w6Wdh0FjnZGlpht2LKf/7Z1gZdzsjxJhU\nFA5M5ay8yAQp6Fw5ZShxlsS9u0+cZuuRqnZGeJyh+D04bUn9lZAG465wTJzOEo2KqCtcCSzXzHJz\nTJPdZcCXRWResIFKqYeUUtOVUtOzs7MjIWsgIy4wUr6Y9JdaLoxZDzhjlvMxPT+T3Ez/7v+G5lbe\n2HKsnREeQUlIgdGXOCvDc5+Fpz5O0dFXSZEGro5dDsC104aFLVNHRwxMS+T80fb/uec1s6FHG+hm\nufGLICE1+LkuIhoV0WHAunEm1+wLxvVoZjml1GHz5wngBQxTnzuJjTd8RRaujV3KwLRE5o12SDli\nlHu+Wotu0ovzeUQJmiK8JnYZQivXnBXUyBAxdLPgyxuOeEExHdFQDdv/a++bcr0zsnSRaFREa4BR\nIlIoIgkYyuZl/SQRSQfOB16y9KWKSD/fe+BiYEtEpO4ulhupRiVSRQpXTR5MfJgzKXTE1Zp57sPi\nk150UzQy7iqaY/yBErlSxq3DjpGbmdLOoPCzcNwgW1DMyRovKKZDtr8CzZb/wf65kD/HOXm6QNQp\nIqVUM/AV4E1gO/CsUmqriNwhIndYTr0GeEspZd1xmQMsE5GNwGrgVaXUG5GSvTvUZI7n0daP8cXG\nb3BWw1/5ZtOXuXKqs7NVgJGD0pica49u+u9GL7op6kjqz9IYu1HgptTVDgnjJyk+lismD7H1vbrJ\nM/+2y7YX7e3J10FMdDzio0NKDaXUa0qp0UqpEUqpn5p9DyqlHrSc87hS6nptXLFSaor5muAb62aW\n7CzlJ4038mbrDBpIYFhGcsCmUqfQzXOvbfYeFG1yeD28+GXY9RY0N3Z8foTYdbyaf9TaFVHRyXeN\n6CuH0aMz3952jMZmzzwXlPpTsPcde9+Ea52RpRtEpSLqS7y6yb7KuGLyEMecyDqXT7LPWDccrORI\npWeeC8qW52HDP+Cf18GvR8Lqh52WCIBXNx1laetkqpU/+CSmphQOrHRQKoOZhVlkWSJDq+qbWb63\nrJ0RfZidr9ur/GYVweBJzsnTRTxF5GJONzTz7k67XfxjmrnCSQanJwWE+HrRc0FQCrZZ3JgNp4xs\nCi7gtc1HaSSexa1n2Q9seyn4gAgSFxvDJRPs5U1e23TUIWlczlbNLDf+alsOQbfjKSIXs2T7cZsp\nYnhWsm3XuRvQV0Wvb/EeFAEc+QhOWTKVxyX5yzY7yO7j1ew+YVQAfr1FCx7d/nJAHSwn+Jh2f721\n7bgXPafT3AAHPrT3Tbg6+LkuxVNELuZVbfb3sUlDDbNcS7NhD66rcEgyP5dqBfnW7q/geFW9Q9K4\nFH11MfJCSEwLfm4EeXWz//56v3UK9WKpOVR9FA6tcUAqO7OKssi0ZJc/VdfE8j2eec5GXCJ8axtc\n9wRMuAZyJsHgyU5L1SU8ReRS6pta+EALV71u0GF4+avwm1Hw92sC9ww4QG5mSkD03JtbPfPcGZQK\nVER6vSmHeN0SXNJAAscHz7ef4Fbz3GZv1R1AQqqxCrrucfjiB1FllgNPEbmWZbvLqG/ymyCGpidR\nVLUK1j8JdWaiiB2vOSSdncsm2s0n3oPCwrHNRpVdH7EJzmdTAA6W17LzePWZdmyMkHWOffM0215y\nRR0b3fy7ZPsJWlqdl8u1REnItpXok7iPsHi7vQz3heNzkHFX2k8qfteobeMwl2nmuTUlFVTWuidE\n2VF2vm5vj7jAqAfkMPr9NT0/k34TL4M4f/QcVYcMReow544YELC5dcNB583SHqHDU0QupLVVBVSm\nvHBcDuRMhHRLdqPmeiPJocMUDExlTE6/M+2WVhVQbrrPsktTRGM/5owcGroiumh8jmHeGbHAfuIu\n5/d7x8diBXmzAAAgAElEQVTGsGDMIFvf29u8yq29CU8RuZANhyopO91wpp2WGMesogGG3XfMZfaT\nd7rDPLdwnP1B4ZV4BqqPGRFzVlwQLXeqrolVxeW2voXjcow3oy+F5CyYfL3hb5j5xcgLGIQLx+fY\n2roi7ZMo5QrTaSjwFJELWbzN/k92/phsEuLMP5WuiHa94YowW10Rvb/zhBdmu+tNe3voWdBvcPBz\nI8j7u0pptvhYRmSnUjjQzNA85Xr4zh649q9GBFaSO7YLnD8621YaYs+J0+wrq2lnRB/gyHr43QR4\n5Vuw+21oit5oVU8RuZC3NUV00TjLbDB/DiRafAw1pXDY+YrnU4dnBuyCX1vSx+34ullLn0Q4xJIg\n/sczxCVCTGSrsnaG9OR4ZhZl2fr079Hn2Pk6VB2GtY/CU5+Al+50WqJu4ykil1FSVnNmkyEY0Uw2\n+3hcgrEPxYoLzHOxMcL8MfbSFH36QdFUB3vftfeNvtQZWSw0tbTy7g672dQ20XExF2py6hO2PsdO\nbaKjPxeiCE8RuQzd9j2jIIt0y4Y+AMbYS4gHRGY5hP6geGdHH/YT7fsgMCW/C3J/rSkpp6q++Uw7\nKzWBaXmZDkrUefT7a01JORU1fTQ6s/IgHLdENEqMK/yP3cVTRC4jaDSTzqiLQCzmk9IdUHkg8LwI\nM3fUQOJj/Xb84rIaiktPtzOiF5OcCeOuMko1g7F3yAWbDBdr0WYLxgwiNsZ5uTrD8KwUxg72R2e2\nKgI2ffcZdLNv7gxIHeiMLCHAU0Quorq+KcCvos8CAUjOgOEz7X17FodRss7RLymemYX2ZJ59dlU0\nfAZ86u/w3WK46T9wzuedlgiAJTv0ic6gNs7EcH7vegte+Sa8+/MwS9Y5FozVg2L6qCLSrSAu8T92\nF08RuYgVe0/aopmKBqaSN6CNSpmjNHvwniVhlKzzBIZx93E7flwijFwIOeOdloSSshr2n/RvgE6I\njWHOqDZKzh9eD78qNMpWrH0MPvq7K0KFzx9tl/eD3aW09rUsC421ULLM3ucposgjIpeKyE4R2SMi\ndwc5Pl9ETonIBvP1w86OdRJ9E+j5Y9p4SECgY7L4PVcUXFs4VrfjV1Bd3+SQNB5WdDPWOYWZpCXG\nBT950Dh7u+ownNgeJsk6z9n5dpnLTjey9UiVgxI5wP4V0OLfZ0hGHgwc7Zw8ISDqFJGIxAL3A5cB\n44EbRCTYdHOpUmqq+fpxF8dGHKVUgJlBn/3ZGDwZ0iwP/aY6OLE1TNJ1nrwBKYzITj3TbmlVrNh7\n0kGJPHx06f6KT4aCufa+vc6vuuNjYzhvpN38+/6uPmb+1c3wIy90hf+xJ0SdIgJmAHvMst+NwDPA\nogiMDSt7S2s4bKlumhgXY2RTaAsRmPwpmHaTkf79u8UwdFoEJO2Yebr5xEv34zgNzS0BEwL97xTA\nyIX2tkvMv/O1dD99Lp2UrohGLAx+XhQRjYpoGHDQ0j5k9unMFpFNIvK6iEzo4lhE5HYRWSsia0tL\nw3+j6/9MM4sGkBTfwcbCi38Ci+430r8nZ7R/bgQJUES7S1Eu8C9EhNJdcHC1UTPKRawrqaCuqeVM\nO6d/oi0/YFD0B9z+Fa5IsqvfX+sPVHKqro+Yfyv2w8nd/nZMHBTOc06eEBGNiqgzrAfylFKTgT8B\nL3ZwfgBKqYeUUtOVUtOzszuYOYaA93bazQvtmk1czqzCAf6URMDB8jpKTjr/AIsIqx+CRy+CXxfB\nv26GAyudlggI4n8cnW0UWWyPASMM/4OPlgZDGTnMsIxkRg3yFxZsaVV9p1iebh4dPtMV2dx7SjQq\nosOAJQU1uWbfGZRSVUqp0+b714B4ERnYmbFOUNfYwqp99iSU0ayIkhNimVFgT8fSZ8xze98xftaf\nMsptu6CKLgQqog7NcmCYf/VVkQv8REBAFg99Itdr0c2juvk0SolGRbQGGCUihSKSAFwPvGw9QUQG\nizndE5EZGN/zZGfGOsHKfSdpbPYnCM3NTLY5/KOReaPtm+v6hCKqKIHyvf52THygw98BjlfVs+OY\nvwhejMCckZ3c/OhSP9H5owP9RL3e/NvSBMXv2/uiOK2PlahTREqpZuArwJvAduBZpdRWEblDRO4w\nT/sEsEVENgJ/BK5XBkHHRv5b2AkWzdSh2cTlzNX2p3xYbFe2vRLfashH3ixITAt+bgTRV0NTh2eQ\nkZLQxtkahfPsWTzKdsKpQyGUrnucU5hJssWHeryqwaZseyWtLXDJvUbGjsR0SM2GHOfTRoWCNjYR\nuBvT3Paa1veg5f2fgT93dqzT6Ps7umyWq6uAfUth3/vGfqKrH4Th54ROwG4wdnA/BvVL5ES1sd+h\ntrGFtfvLmT0ietOQdIi+WtCLzDlEt8xyPpLSjSwRBz709+1ZAmffGiLpukdiXCyzRwxgiSVzx/I9\nZYwbEv3+kjaJT4KzP2O8Wpqhcn9UlgUPRu/4FlHM0VN1FJf666rExgjnjmgnbDsYr3wLnr0Z1jwC\nJ/fAvvdCK2Q3EJGAVdEHu3qxQ7mlyUh0asUFYbXBHPldnujo38MF6aQA5oyyT2qW9ZWABYDYOCOY\npJfgKSKHWb7Hvrdj2vAM+iXFt3F2GxTNt7d1O7JD9Ck/0eF10GDZ4Z8y0Nh07DDbjlRRWesPbU5P\njmdybhdD/UdeYG8Xv++K8HTdz7WquLz3m397KZ4ichh9tjq7s05kK0Xn29sHV0Gj89Ur547Ktm34\n3na0ylYCvVcRzCznArPJ8r32++vcogFdz7Y9ZJpRPtxHwyk4uiEE0vWMkYPSGNQv8Uy7rqmFDQcr\nHZTIo7s4/5/Sh1FKBZgTOh3NZCWzwHj5aGm02/QdIis1gYlD7aWmP+yt6X70QAUXmOUgcKKjp8fp\nFDEx/k2TCf3MAn/OB9OICOeN7MPmuV6Ep4gcZM+J05RW+1cIyfGxTB3ezQwJRfPt7eL3uitWSJmt\n+btW7O2FD4q6Sjiy3t7ngkCFhuYW1pTY96fpD+5OM+ebcNvb8L198Ol/Qe7ZIZCw5+jfp9dubP3w\nL8bm6JbemUHCU0QOos/eZhZl2TISdImi+fa2S/xEuqmxVyZAPbASlMU3kT0O+g12Th6T9fsrqW/y\nyzUkPYnCgd3cnzZ0qhE9F9tF/2WY0Vd4Gw5W9r5s76cOwZvfh8cugV8WwDM3uqIkRyjxFJGD6IEK\n3TLL+SiYh81ccmwT1Dj/0D+nINNWtXX/yVoOVfSydD8lS+3tQuc3sULg6nP2iIFRvz9NZ0h6MkVa\ntvfVWpaSqMc6qWw8DbUnoz7bto6niByiuaWVVcV2RdGjPTapA2CIFqW1z/lVUUpCHNPyMm19K/Y4\nryBDil6krGCOM3Jo6CvubvmHogB9Atfr/ET6/3Hh+cHPi2I8ReQQmw6forrBHwI7IDWBsYM7yIbc\nEUXz7W2X+InO0xSsHskV1TSchuNb7H355zkji4Xq+iY2HTpl6+u2f8jl9Go/kVKB/8d6lGwvwFNE\nDrF8txZWO2IAMV0Nq9XRZ0ouUUSzR+oBCyd7T16wxDT49m64/p8w604YvwhSnX/gryoup8VSQnvk\noDRy+ieF7gItzXBonaGIHWZW0QCs/zq7jp/mRFW9cwKFktKdcPq4vx2fCsOmOydPmPAUkUPoq4Ie\n+Yd85J0LsZYcYpX7oXxfzz+3h0zJzSAlwZ8XrLS6gT0nnH+AhYyULBj7Mbj05/DJJ52WBgi8v87r\naraOttj8HDx9A/yqCB65INA/5gDpyfFM0jbp9pqgGN0slz8b4jqZJzCK8BSRA9Q2NrN+v33jXUjM\nJgkpRn0SKy7wEyXExTCj0F4WoleZT1xISDZKB+PgKtj5mrGpFVwTnTlHW3X3Gj+R/vvthWY58BSR\nI6wpqaCxxR9Wm5eVwvCslNB8eNF8e9sl5rlAP1EvmbG6kBPV9ew67l9xxgjtl53vCrr51wUTHQju\nJ4p6829Lc2AgTC8MVABPETmCHlYbUidy0XzjZ3ImjLsSRl0cus/uAbqfaGXxSZpbvLxg4UDPXjEp\nN4P05BDt/yk4D8Ty2DixDU47X5TurLxMEi178I6equdAeZRvEzi60b/yBEgZADkTnZMnjHiKyAFW\nFdv3OXQ523Z7DJkCdyyH7xTDp/4BUz8dus/uAeMG9yczxf8wrK5vZuuRqnZGeHQXPTw+ZP4hMCY4\nQ6bY+/Ss4w6QFB/L2fn2bQIri6N81a1n0S+Y64r8heEgKr+ViFwqIjtFZI+I3B3k+I0isklENovI\nChGZYjlWYvZvEJG1kZUcTjc0s/mwPax2VlFWG2d3g9h4GDzRdTdsTJDyFlEfxv3srfDfbxgO/Opj\nTktzhpX7Qrg/LRguNc/p5seoz2uom+V6qX8IolARiUgscD9wGTAeuEFExmun7QPOV0pNAn4CPKQd\nX6CUmqqUingc5NoSe1htUXYqg/qFMKzWxegPxKh+UNRVwraXYN3f4Pnb4LdjoNb5Hf1HT9Wx/6Tf\nJBUfKwErhR6jPxD3OR85B4GWhZXF5dHrJ2ppggOr7H0uKDsfLqJOEQEzgD1KqWKlVCPwDLDIeoJS\naoVSqsJsrgRyIyxjm6zUzHIhcyJHAXoC1LUlFTRFq5/owIeA5SGXPc4I43YY3ew7JTeDZEvofEgY\nPhNiLD6nin1w6nBor9ENJuemkxTvf6Qdq6q3KeWo4sgGaLKUcknLgQEjnZMnzESjIhoGHLS0D5l9\nbXEb8LqlrYDFIrJORG5va5CI3C4ia0VkbWlp6Aq6rdLMJn1JERUOTA2oH6Pv/o8aXJrWR/eLzAyl\n2ddHQioM07Jv718e+ut0kcS4XuQnKtsJYplAFMzpdfnlrESjIuo0IrIAQxF9z9I9Ryk1FcO092UR\nmRdsrFLqIaXUdKXU9OzsLpZWboOahuaAB++swjDPopsbjSX+kY/Ce51OICLMLAqMnotK9I2cLlFE\nq/ZFaMWtf18XbGwFmFWo+Ymi9f6adhPcvR9ufN4owTHx405LFFaiUREdBoZb2rlmnw0RmQw8AixS\nSp25G5VSh82fJ4AXMEx9EWHt/gq7f2hgKoNCmXbFysE18OTV8Mt8eOxiWPb78Fyni+iBGfqDMyqo\nq4Sjm+x9Lsgvd7yqnn1lfnNOXEwY/EM+CrTvq68QHWJWgJ8oitNJJfaDURfChfcYmTt6MdGoiNYA\no0SkUEQSgOuBl60niEge8B/gZqXULkt/qoj0870HLga0jJXhQ8+2ra8OQkpMDBS/C02mjbxkmStq\nmMws1P1E5dHnJwrmH0oLzaq5J+iry8m56aQkxIXnYsNnQozls8uLoepIeK7VBabkZtj8RMerGiiJ\nVj9RHyLqFJFSqhn4CvAmsB14Vim1VUTuEJE7zNN+CAwA/qKFaecAy0RkI7AaeFUp9UakZNcfFCEN\n29YZPMUo6+yjtgxKd4Tvep1kRHYqA9P8fqLaxpaAcHbX41r/kH11GdaJTjA/UYnzfqKEuBim59v/\nr6LW/NuHiDpFBKCUek0pNVopNUIp9VOz70Gl1IPm+88rpTLNEO0zYdpmpN0U8zXBNzYS1DYG8Q+F\n80ERGwf559r7XGA+EZFA81xxlJnnXOsfinAgjFv9REWeIoo2olIRRSPr9lfQbPEPFQ5MDW1a/mC4\n9EER1QELLvUPnaiqp7jU7x+KDad/yEfBXBg8CWZ+CT71lOHLcAHBNrZGjZ8oWuQMMWEyIHvoRNQs\n50PfAOfzEzkcBnqu9t3XlpTT3NJKXGwUzItc6h/Sgz4mDUsnLTHM/94jFsAI51fZOpNzM0iOj6Wu\nqQWAE9UN7CuroSg7zWHJOsGRj+Dft0L+HGMiWTgXMvKclirsRMF/fu/AkY2sgydDYn9/u/YknNge\n/ut2wIjsNAam+Wuq1DS2sCVa8s651j8Ugf1DUUJCXAzTC/T9RFFi/i1ZBpUHYOM/4aU74c3/dVqi\niOApoghg+Ifs9Yf06LGwEBtnFMuz4hI/kf79o8Y851r/UN/N2BEM/ftHzf2lbwx2yf0VbjxFFAHW\n76+kqcVvzikYkMLg9Ajll3OpnygwYCEKHhQu9Q/pFW9jBKaH2z/kcoIFLLjeT9TaAvtX2Ps8ReQR\nKgL9QxGcrRYG8RO1Or9vR/8drCmpcH99ooYqmHA1pJo+Idf4h7T6Q8PS6ZcUovpDUcqkYYafyMeJ\n6gaKLZt9XcmxzcY95iM507jH+gBesEIEcNR+7/MT+W7wunIo3Q45EyInQxBGDkpjQGoCJ2saAaM8\nxtYjVUwZnuGoXO2SkQfXPW4EfJTuNHxuLkAPf3fELFd9zJjklCyDsVcYGQEcxOcnWrrbX2pkZfFJ\nRrg5YEE3m+ef57pyLuGib3xLB6lrbGGjE/4hHzGxkD/b3ueCtP1G3jk93Y87HuwdIgKDxgamuXEI\nxwMVlv3OKIPx/G1GWYydr0b2+m0Q6CdyecBCH/UPgaeIws76AxU2/1D+gBSGZiRHVgg9jHu/8wEL\nEIUPChdSdrqB3bp/qCDCikgvX+2CgBgIHrDgWj9Ra4uniDzCR4B/KJKrIR/6zP3AKldsnNNXhmv2\n2YsGenTMai1absLQdPpH2j80fKa9ZEHZLqg+HlkZgmDk2vPLVepmP9HxrVBvybySlAGDnDWfRxJP\nEYUZ3X7vyP6OwZON8s7nfQNufA6+us7xTa0AowalkZXq309U3dDMtmjZT+QS9GjDiGyU1knqD0On\n2vtcUJ8oPjYmILuEa9NJ9WH/EHiKKKzUNbaw4aDmH3LCkRwTC7e+DBf9CEZdZDw4XEBMjDCjIEry\ngh35CJobnJYigIBEp06suCHINgH3mudcSYBZzh3+x0jhKaIw8tGBChotIcl5WSkMi7R/yOVERYLK\nukp4+AL4RR48fgW8/ytXmDbLaxrZebz6TFsEzgl3ocW2yNcUkQtWRAAzCwMDYlznJ2pt7dP+Ieik\nIhKRKeEWpDeycp8+W+27aVfaQi9ktrrEhX6iAytBtUJzvbEheOsLrjBtrtaiDMcP6U96skP7h/Jm\ngVgeJ6U74PQJZ2SxMDka6hOd2AZ1Ff52YnpgAEgvp7MrorUi8pCI5IRVml6GoxtZo4TRg/qRkeJ/\neFbXu9BP5NK0Po7kL2yLpP4wxH1+oqioTxTgH5ptmNP7EJ1VRI3AbcAuEblbRBI7GtDXqW9qYcMB\n3T/krYh0YmIkqPnEVURLolOnV9wB5cOdV0QQxDznNkU0aCxMvh765xrtPuYfgs4rotHA00Aa8FNg\nu4h8MmxSdYCIXCoiO0Vkj4jcHeS4iMgfzeObROSszo4NFR8dqLT5h4ZnJZObmRKuy3Weiv3w0VPw\n4p3wxvedlgZw+X6iuko45r78cpW1gf6hGY4roiDppFyAbv5dWVzuLj9R0Xy49q/wzS3w9Y2GUnIY\npRTr9pfT0NwSket1KsWPUuowcJOI/An4HTALeFpEvgZ8Uym1Jowy2hCRWOB+4CLgELBGRF5WSm2z\nnHYZMMp8zQQeAGZ2cmxICJytusAsd+QjeGi+v52aDZf8zHF/h/67Wb3vJC2titgY5/0wZ/xDPgaN\nh9SBzsljsmpfuS1eYuzg/mSkJLQ9IBL4/ES+31fpdqgpc/z3NTk3ncS4GBqaDbmOVdVzoLyW/AGp\njsoVgAhkFjgtBQAlJ2v5+AMfkhgXw7S8DOaPGcQd548I2/W6FDWnlFqllJoN3AQcBmYDK0Xk7yKS\nGw4BgzAD2GOW/W4EngEWaecsAp5UBiuBDBEZ0smxIcGV/qGcSZBgybVVUwplu52Tx2Ts4H42J3tV\nfTPbj7rET+RS/1BgfjkXmH2T0o09a1Zc4CdKjIsN2E/kOj+Ry/CZLxuaW1lZXM7S3aVhvV63wreV\nUv/EMNf9CKgDbgR2isiPRCTc9qdhwEFL+5DZ15lzOjMWABG5XUTWisja0tKu/RHqm1r4SN8/5LTZ\nBMz6RLPsfS4oCxETIwFmJb2+jmNEjX/IBRMdiKL9RC65v1yK/v8X7vur2/uIlFL1SqkfYSikp4Ak\n4P+A3SJya4jkcwyl1ENKqelKqenZ2V1L9V9R28jMwqwzaeiHZSQzPMsF/iEIfFC4YMYKLt146FL/\n0KnaJrYfs68YXTHRAfv9lTIQ4iJUd6sDggUsuMpP5CKUUgEBHeG+v3pcBkIpdQS4WUTuxwhoyAce\nE5GvYviPQj3lPgwMt7Rzzb7OnBPfibE9Zkh6Mn+/bSaNza1sPnyKCrPUgSvQNx6WLDM2ZzrsJ9JN\nS6v3ldPaqohx0k/kUv/Q6hLdP9SPzFSH/UM+8mfDx35r3GfZYxy/r3xMGZ5h8xMdOVXPwfI68gY4\nOEGsOmr40YbPhAT3+KsOVdRx5FT9mXZCXEzYy7N0e0UkIiNE5FMi8hsReR94C8jzHQbOAt4TkWdE\nZGgIZPWxBhglIoUikgBcD7ysnfMycIsZPTcLOKWUOtrJsSEjIc7IdXXheBdtvxo6FeItN/3p43By\nr3PymIwd3J/+Sf550am6JnYcq25nRARwrX/Ihf5HH0npcM7njZBklyghgKT4WKbl2R+mK53eJrDj\nFfj7NUbGjkcugg1POyuPiW6NmDY8g6T48O5r6mxmhVwRuVpEfioib4rISWAX8E/gm8BcjNDuemA5\n8FvgXqAC+CSwRUSuCYXASqlm4CvAm8B24Fml1FYRuUNE7jBPew0oBvYADwN3tjc2FHJFDbHxkDfT\n3ucCP1FsjDCj0GXmObf6h/a5bP9QlOA686/PLN7aDIdWw+ljzspjEuAfisBEp7Omuf2W99Zpzl5g\npeW10XzYGyeK/Bb4GfAl4N8icoVS6o2eiQxKqdcwlI2170HLewV8ubNj+xwFc2DvO/52yTKY/lnn\n5DGZVZTF4u3+8gGr9p3kc3MKnRHGrf6huia2apknHN8/FCUYisgfJepoJm6lgkx05gY/N8LoG8pn\nReD+6qwiEqAKw7R1RvEopdqdUiilTgFfFpGdwO+B/wV6rIg8ekiwBJWu8BPZZ16rnPQTudQ/tFbz\nD43J6ceANC/RSWeYOjyDhLgYGk0/0eHKOg6W1zoTSFS2y9g+4SMhDYY4n9LT+J3UnWnHxwrT8jLb\nGREaOusjmgRkKqUuUkr9QCn1akdKyIpS6o9ANTC5o3M9IsDQaRBv+eerPgrlxc7JYzJuSH/6WfxE\nlbVN7DrhkJ+ocB7c/CLM/TYMnwUjLnBGDg3Hy4J3h+YGqHU+XDopPpZpmtPdMfOcvhrKm2WYzR1G\n9z9Oyc0gOSH8ee86pYiUUltVz2MdKzD8SB5OE5dgROpYccF+j9hg9Yn2OvSgSEiBEQtg4Q/gtjfh\n4nudkUNDt9+7KlDBysm98N4vjbIZv8gzSme4AN3f4dh+tWCF8FyAU4U8I1mP6HaMDbAebiAgQaXz\nighcvPHQBRFgVfVNbDl8ytbnWv/Qie3w3s+MQJjmetjvlvvLBZm4lQpSf8id/qFIbZSOmCJSSr2l\nlPpxpK7n0QH6je/zEzmMrohWlxh+Ig9YV1KB9VcxalAaA93qH8qfbW8f2+IK89xZeZkkxPofe4cq\n6jhUEeH6RCf3GNsmfMSnBpZad4DjVfW2Wk2xMRKQGilceBVa+ypDz4I4s1ps/2FGJF1TXftjIsD4\nof3pl+j3E5XXNLL7xGkHJXIPrsxf2BYpWVpxNwUHPnRMHB9J8bFM1fxEEY+e07dL5M10hX9Iv78m\n56aTmtjjnAedwlNEfZW4BPjkk/C1DfDNrXDtQ4ZfxGFiYySg3LXr6hM5RNQFKuh+D5fUJ3LcPKf/\nHlyyPy3S+eWseIqoLzP6YsgqdIX/w4q+QTOiD4q6Snj+C7DuCcPh7gJzJUB1fRNbjuj55Vy8IoIg\nCVCd3zgNDgcsuHn/kIMTncisuzw8ukDAfiKzkJlEQmEe+BA2P2u8wChadstL4b9uB6zdX0GLxUE0\nIjuV7H4u9Q/50FdExzYbij45vHnLOuKsvEziY4WmFuP3eaC8liOVdQzNSA7/xcuL7RkU4lOM7RQO\nU1rdwN7SmjPtGIHpEfIPgbci8nAhE4b2J81imz5Z08ieSPmJ9NnqwNGRuW4HBNYfcvlqCCB1gLER\n+Azu8BMlJwTxE0XK/KuvCoe7wz+0WlsVThyWTr+kyMnlKSIP1xEXG8P0AocKmbk00WlUBSpYcWl9\nIt2suXJvhMxz+zVF7JL7KzBsO7L+R08RefhprDH2f7iAgP1EkbDj11XCUffll6tpaGaztn/I9YEK\nPlyqiALvrwhNdK76I3z2DVjwf4bZt2h+ZK7bAU4XWvR8RH2dugpY/kdjH9HhddBvKHxzs9NStVnI\nLKx+ov3LAUtwglvyy2n+oaLsVAb1c0fBuQ4J8BNtcoefKD+DuBih2fy97j9Zy9FTdQxJD7OfKC4R\n8s81XnwnvNfqJGWnG9h13G/6jhECIlfDjbci6uvEJcOH98PBVUY6+lMHoGJ/x+PCzMRh6aRaclyV\nnW5kb2mY/UT7dLOcO6KZnJ6t9ojUgZA9zt9WrUZCWYdJSYgLKPbmaDZuB9HvrwlD00lPjqzfylNE\nfZ34JMidbu9zQfnw+NgYput558L9oND9Q4XuUESBhfCixCznI6A8vVvMc95+NYAPtXyO546I/ETH\nU0QerrXj636QsAYs1JyE41ssHeIK/1BtYzObDtn9Q1ETqODDpXkNAwIW+uiK6MNiTxF1CRHJEpG3\nRWS3+TMg0F1EhovIuyKyTUS2isjXLcfuEZHDIrLBfF0e2W/gUlyqiILVJ+p5Evg20GfpgycZaWoc\nZt3+ijN+DIDCgank9I8S/5CP/DkQm2CU05j7bbjwHqclAuDs/EziLLWu9pXVcLyq3kGJIs/xqnqK\nLfuHYmOEcwoif99HlSIC7gaWKKVGAUvMtk4zcJdSajwwC6Mwn3Uzw++UUlPNV9+u1Ooj9xzjQeGj\ncj9UHnBOHpNJw9JJsfiJSqsbKC6raWdED9D9Q4XzwnOdLhIYtu28cuwyadlw90GjnMbCH7gmUiw1\nMSHml04AACAASURBVI5Juem2vrCturf8B569BVY9BMe3QWtrx2MiQLD8cmkRyi9nJdoU0SLgCfP9\nE8DV+glKqaNKqfXm+2pgOzAsYhJGI/HJMEzzE7kgL1h8bExA9t+wPSgC9g+5xT/kXP6vkBLvzlVc\nxMqO7HoTtr0Er38HHjgXlv8uPNfpIiv2aGY5h8y+0aaIcpRSR833x4Cc9k4WkQJgGrDK0v1VEdkk\nIo8FM+1Zxt4uImtFZG1paWlbp/UedDu+axzKgel+Qs7pE1C6w9+WGDO81llqG5vZeKjS1hc1+4ei\nhEDzbxgmOsHyyw2fFfrrdAM3+IfAhYpIRBaLyJYgr0XW88yKsW06DEQkDXge+IZSypct8gGgCJgK\nHAV+29Z4pdRDSqnpSqnp2dnZPf1a7se1fqLAgIWQ+4n01dCQqZCUHvzcCLKmpOJMPjSA/AEp4d/n\n0sc4Oz+TWIufqLi0hhOh9hOVF0PVIX87NhGGnR3aa3SDw5V1HCj31x+KjxWm5zsz0XGdIlJKXaiU\nmhjk9RJwXESGAJg/TwT7DBGJx1BCTyml/mP57ONKqRalVCvwMDAj/N8oSsidATGWvQMVJXDqUJun\nR4pJwzJIjvf7iU5UN7Av1H6ilibILPC3XeIfWrG3zNaePcL5zbW9jbTEOCYN0/xEoc7ise99eztv\npitMlXrY9tThGSRbfLKRxHWKqANeBm41398KBKRFFmPr/aPAdqXUfdqxIZbmNYA1Xrdvk5ASOEtz\ngZ8oIS4w71zI0/ZPuR6+vhG+sRmufgAmXRfaz+8muv3+vJFR6h+yopRRXmP9k4YD3wXo5k5931aP\n2feBvV14fmg/v5sE7B9ycFtAtCmiXwAXichu4EKzjYgMFRFfBNx5wM3ABUHCtH8lIptFZBOwAPhm\nhOV3Ny7deKin+9H/gUJGRh5M/TQMntjxuWGmsraRLUfs+4ecfFCEhIOr4b7x8Kez4OWvGhk9XEBg\nwEII76/W1kBFVDQ/dJ/fTZRSgRGZDvmHIMpyzSmlTgILg/QfAS433y8DgiYkU0rdHFYBo52C82Dp\nb/xtPaTZIfQHxYq9Ecg75zAri8ttNfnGDu7HgDSX1x/qiIx8qD7ibx9ZD/WnHPfHTTf9RL58fntL\nazhRXR+afH4ntkKt5YGf2N/wQTrMwfI6DlfWnWknxMVwVl7k6g/pRNuKyCOcDJ+p+Yn2uWI/0ZTh\nGVreOXuSxt5Ir/QP9csJzDvngqCYfknxTBza39YXslW3vhrKPw9inZ//f1hsv7/OzsskKd4Z/xB4\nisjDSkKqYZ4rmAsX/B98fomRjdth4mNjAso7L9tT1sbZvYMVe3uhfwgCzVLF7wc7K+LMHmlX9MtD\ndX/p36/Ipf4hB81y4CkiD52bX4DPvALzvmMkQ3XB7A3gPO1BsSIUD4rGGsM05DKOV9XbKtLGxggz\nIpyWP2zoD+Li9xwRQ2eOdn8t213W820CLU2BCYRdEKiglHLN/iEfniLysONSv4u+IlhZfJKmlh6m\nSdnyPPyyAB5aAG//EA6t69nnhQh9tjo5N7Jlm8NK/nkgFhNQ2U6oOtL2+RHi7PxMEuP8j8Mjp+op\nOVnbzohOcOQjaLSYkFOzYdC4ts+PEHtLT3O8quFMOzk+lim5ztaH8hSRR1QwJqcfA9P8+fBqGlvY\neLCynRGdYO+7hp/iyHpY/gfYs7iHUoYG3Sx0Xm/wD/lI6h9YdsQF5rmk+NiAZJ89Nv/q36twnism\nekt327/XjMIsEuKcVQWeIvKICkQkwDzXowdFa2vgRsMRC7r/eSFCKRXgH5rtsNkk5OjmKf3v4BD6\n/bV8dw8Vkf69XGCWA8PsaGXuKOcnOp4i8ogaAv1EPYhsOrZJC6tNh6Fndf/zQsSB8trAsNp858Jq\nw0LRfHu7+D0IV3mPLqD7iVbsLbOVaO8yQ6dC9lh/2wWBCk0trQH7h+Z4isjDtSgFpTth1V/hXzdD\nc0PHY8KMrojWH6igpqG5ex9W/K69XTjXFYEZyzXlOj3f2bDasJB7DsSn+NvVR6Fsl3PymEwY2p+M\nFL8vrqq+mS2HexDMcvG98OVVcNdOuO4Jexoph/joQCU1jS1n2tn9EhmT089BiQw8ReQRnIcvgPtn\nwOvfhe0vw6E1TkvEsIxkCgemnmk3typWdzfdz15NEbnALAewXNs/pCvfXkFcAuTPtve5wE8UEyMB\n/riQbBPoNxgmBFSscYRlu+2VBOaMHOiKjeGeIvIITvYYe9slYbZ69Fy3HhRNdXBgpb2vyHlF1NKq\nAgIVnA6rDRtF8+1t19xfYdpP5BKWat9HN0c6haeIPIJTNN/edsmDQv/H6daDYv8KaLGYGjPyIKuo\nh5L1nM2HT1FZ23SmnZ4c73hYbdjQHfclS6Glm2bWEKLfX2tLKqizmLKimVN1TQGRpm7wD4GniDza\nQn9QHF7nis2fs4oG2CJgdxyrprS6i/6rve/Y20ULXBFW+8GuQLOJtVZOryJnIqRYVnup2fY8dA6R\nNyCF4Vn+mk+NLa2s3R+mqq0R5sO9J7HGXozOSSOnv/PlKMBTRB5t0X+IPeLHJXnBMlISAurH6HnZ\nOkRf3bnEP6Qronmj3TFbDQsxMXD+3bDofvjGFvjaemNl6gICsix0ddV9YCVUHw+hRKFhaYB/yD0F\nPz1F5NE2RfPtbX0l4RC6Hf/9nV0o5X76BBy3lqESV+zvqKpv4iPNbDJ3lHseFGFh5u0w7SbIGO60\nJDb0+2vpri4oIqXg2Vvgt6PhgTnw1g9cYUmAQIXqhv1DPjxF5NE2ugN/99uu2O9x/mj7A/qD3aW0\ndna/h74aGjoVUpzP47Ziz0nbnpWRg9IYmuGVBXeC2SMG2iy1245WcaK6k+XDj2+B0+Zq6PhmWPsY\nxDn/dzxYXsv+k/ay4HpBQCeJKkUkIlki8raI7DZ/Bt3pJyIlZgG8DSKytqvjPUwK50KsP60OlfuN\n6poOc3Z+JmmJ/j0/Zacb2XqkqnOD9bBtF0TLgaFMrczr7ashF5OVmsBkzfzb6VX3niX2dsFcI1zd\nYfS0PmflZZKS4Py+OR9RpYiAu4ElSqlRwBKz3RYLlFJTlVLWxFZdGe+RkBq432PP287IYiE+NiYg\njPv9XSc6HqhU4EZWF/iHlFIB/qG5vdk/FAWcP2aQrf3erk4qIt18PTKgjqcj6P4hN5nlIPoU0SLg\nCfP9E0BXd4n1dHzfY+RF9vZu5xURwHz9QdHZGestLxk73gvmGpVBh88Mg3RdY19ZDYcq7Gl9ZhX2\n0v1DHXH6hCvMv/PH2FekS3eV0txRtvfGGjjwob1vxAUhlqzrNLW0BqyI3OZ/jDZFlKOUOmq+Pwbk\ntHGeAhaLyDoRub0b4xGR20VkrYisLS3tgjO8tzFKU0T7lxsbQh1G9xOtP1DBKcsenKCIGBt1Z3/V\nqLl01y6Ic778tr4amlGQRXJCL0vr0x4HV8PiH8GDc+E3o7RgEmeYkptBppbuZ0NH2d5LlkNLo7+d\nWQADRoRHwC6wpqSc05ZUWAPTAiNPncZ1ikhEFovIliCvRdbzlFG1qq2p0xyl1FTgMuDLIjJPP6GD\n8SilHlJKTVdKTc/OdtfsIaIMHA3plrDa5npXhHEPzUhmdE7amXar6kaYbbw79lB84MJsyBFl6X2w\n7D4jES24ohxHbIwErBo6XHXv1fxDI9xhltPlPn/0IGJctj/NdYpIKXWhUmpikNdLwHERGQJg/gzq\nGFBKHTZ/ngBeAGaYhzo13sOCSKCd2yXmOX1V1Ck/kcuoa2wJyA4xb3Qfm/iMutDe1h3+DqGb595r\n7/5SCna9ae9zgVkO4J0ddrkvGDuojTOdw3WKqANeBm41398KvKSfICKpItLP9x64GNjS2fEeQbCa\n53ImQVahc7JY0P1E7+8q7Xl55wizYm8ZDc1+38PQ9CTGDnY+G3JE0VcOBz6EhmpnZLGgTwi2HG4n\njLtsN1Ts87djEwL34TnAwfLagLLzbknrYyXaFNEvgItEZDdwodlGRIaKyGvmOTnAMhHZCKwGXlVK\nvdHeeI8OKDwfrvozfGsHfGkZzPqS0xIBML0gkxSLL+V4VQM7jjn/AOsKS7TZ6sJxOa7IhhxRsgph\nwEh/u7XZFdm4B6YlMjnX7kv5oK3Nrbtet7cL5kJiWvBzI8h7O+3319n5maQnu6/sfFQpIqXUSaXU\nQqXUKNOEV272H1FKXW6+L1ZKTTFfE5RSP+1ovEcHJKbBWTcbaX9cRGJcbED1Ut0MAUBrCzz1ScMX\ncXyrK6KywAjbfme7ZjYZ5z6zSUTQozN3vRH8vAgzf7TuJ2rDPKeb5UZfGiaJukY0mOUgyhSRh4eO\nbp57e1uQHF+H1sLuN2HJj+CB2UZ0lguU0dYjVRyr8pt6kuNjObeoj4Ztj77Y3t71hjGBcBh9P9EH\nu0pp0sO4a8sDy4qMviTMknVMfVNLQNn5BWM8ReThEXIuHGePwN9wsDLQjr/7LXs7e7Qrsm3rs9Xz\nRg7sfdVYO0v+HEjs72/XlBoZ3x1m6vAMslL9mRGq6ptZoxdj3LMElEVpDhoPmfkRkrBtPtx7MsD/\naI00dROeIvKIaganJwXY8Zdo5i52a2aTUc7PViHQP3RhXzXLgZEGZ6QWPbfjVWdksRAb8//bu/Pw\nKKqs8ePfk85GQiAQ1rAFhLCFPYCAgBh2UBa3ccF9GV9AHZXRGVFh3lGZcUSdV4ffuKCgOCo4CiqC\nICKyCiRskX0PCVuAANmX+/ujOklXd5osJKkuuJ/n4YGqrkpOQ5NT99atc8RjOusH91F3crx52wdG\nQ+B5oTOoXQOfvf+oE5FWMWcPw54fSj+uGgxxGxWZpufOHYXj211eLWE5ugVOXcj2aFI2yEfn76tN\nu1Hm7d3fl3xcNRvawfPzZVqdOfxVmLQZhr1iLFJo6/Y+LKCUYvlOc8J0n8b2JToRaWWXkwE/vQKz\n+sFbneHLByEvp/Tzqthgtx8Uq/edJr3wSXL3q+pmvSDU+uWrP7nd9O7UpLbPNCmzTOvB4OdSiPP0\nbp8ostu/TX2CA4p/VB47l+lZZLdea+gz0ajY0axnNUfoafuxNFLSiqeogwP8fKYteEl0ItLKzj/I\nKGtfWIIl+7zR4tli7RqF0bSOS1fNPJfaWru+dTt4dDVG5t2PblercVfztFyhGuHQop95nw9Mz9UI\ndHhUWShxUYwPWZp43LQ9MLq+T5eN0olIKzs/B7QdYd63e3HJx1YjEWFICdMnpKcatfFctbc+EWXk\n5PGzW305X11WW+3ajjRv+8j0nPvny+M+kY/5IdEc37COjSyKpGx0ItLKx33+e9din1gK7f6DYsWu\nE+TvWmy0OC/UoCPUbVXNkXn6efcpsnLNq5l8rQilZdq5JaKj643l0RaLa9cA1/JsO1POc/RMhvcT\nLHTg1EX2ulVTiGvntb6zT9CJSCufVgMhILR4+0IyJCdYF49Tz6i61Aouvr9wNiOX8wlfmQ/ygdEQ\nwOId5mmT4TGNfXY1U7ULbw6Nuxo3/Ye9CpPjfaKDbkTNIGJbmOP4adtBL0dba6nbaOjaVnWpHeJ7\n1RRc6USklU9ADWjtVszRB+bxAxx+pumtELIIO+Z2/8oH7g9l5eazwu3+0IhOvj1tUu0eWm7c9O/z\nPz5T1xDcR92KuNW3w78Hwi+vw+l9lsXl7offzBc6vj4tB+A7vWJtJjc3l6SkJLKyytjL/krS7klo\nPsG5oQjOPEHTnBwCAq1tiTysYyO+3pIMwEC/rfgrlxV94c2hUSeLIiu2eu9p0nOKH35sEBZEj+a6\nY72Jwzev3od0aMjLi3cC0EaO0STvKKQchZQtxmrSPx6E4FqlfJWqdTwti4Qj5scChnbQieiKlZSU\nRFhYGFFRUVfftEpBHhzfASiUUqSm1yVp/05atu9iaViD2jUgJNBBRk4+ox1unTLb3egT1RS+3+F5\nteprvWG0kkXVC6VdozB2Hb/ACL9fzS+2ut7yJATw7bZk03aXZuE0qu37jwXoqbkKysrKIiIi4upL\nQmA86xFktCoQESJC/cnKTLc4KAgOcDC4fUNqkkGcn9t9q47jrAnKRU5eAcvcpk1GxPj+1apW7MYu\nkQCMcGwwv9D+Jgui8fTNVnMiurGzbxUq9kYnostwVSahQjWKp5NExGgf7gOr50Z3bkwBfryadyeb\nCqIBULWbQdNYiyODNftOcz6ruGVz3dBAerW0/ka8bRQUlH5MFbuxcyTXyDHa+x0t2qfE4RP3Hw+n\nprM1Ka1oWwRGd460MKKy04lIq5jg2oBLIi7IM6pcW2xg2/o4gmoyJ38Yt+RM47rst0js9TefmJb7\nKuGYaXtYx4b4O/R/wUs6vRdW/g3euRYSPrY6GppHhPBIHXMx1uTwHhBqfdX0b7elmLZ7RtW1xbQc\n6ESkVZSfw3NOPPG/1sTiIsjfwZCOxaubklR9Pj3Z3MKIDBez8zxWM43t2sSiaGzi1/fg7VhY+Qqc\n2ukTny+UYmTBKtOuL3P7WBSM2aItbtNyXewxGgKbJSIRqSsiy0Rkr/N3j+VGItJWRLa4/DovIk86\nX5smIsdcXhvp+V20Mqvh9tef+JVP9JAZ7TYvvnh7Ctl51sa1dMdx00OsTcJr0DNKT8tdUssB5u2D\nq+B8csnHVpcj6wnLKo4hSwXw3ulOHE619h7p7uMX2H2iuDuxw08YaaP7j7ZKRMBzwI9KqTbAj85t\nE6XUbqVUV6VUV6AHkAG4Ptn4RuHrSinr69NUgb59+5Z6TGZmJgMHDiQ/3/sP6JycHAYMGEBeXl7J\nBwTVAnH5CNVrAxmpJR9bja5rXZ9wlwf4zmXkenRCrW5fbzFPy93UNVKvlitN/bbmJfeqALZ9bl08\n4PH9lxf04AIhfJ1gbYJc6Pb56te6HhE1gyyKpvzstnx7DHC9889zgJXAs5c4Pg7Yr5Q6XFUBRT1X\ntQ9zHppR/pLya9euLfWY2bNnM378eBwO74UQAwMDiYuL4/PPP+euu+7yPMDPAaH1AYFa/nDvN+WO\ntSoE+vsxpkskc9YV/7Mv2JzEiE7WrCA6eT6LNftOm/aN66an5cqk612wxOV6c8un0O9Ja+755WUb\no34XX+UbRVrnbz7K5BtaW3JxkV+g+DI+ybTPLqvlCtltRNRQKVV4R+44UFoBpd8B/3HbN1lEtonI\n7JKm9gqJyCMisklENp06dcrbYZZKT09n1KhRdOnShZiYGD7/3Lhaq1mzJocOHaJ9+/Y8/PDDdOzY\nkaFDh5KZmVl07rx58xgzZkzR9qBBg1i2bBkAU6dOZfLkyQCMHTuWefPmeQ+iViTUamwu32+VtW/D\nDy/AqT3c0qOZ6aWVe055dm6tJou2JlPgsqCwQ+NaRDcMsyQW2+l0q1triD3WdW7duwyyih8WPavC\nWFVgPDuXdDaT9QesmQ1YtecUJ85nF22HBDosu+iqKJ9LRCKyXER2lPBrjOtxyuhM5XW9sIgEAjcB\n8112zwJaAV2BFOB1b+crpd5VSsUqpWLr16/v7TBLLVmyhMjISLZu3cqOHTsYPny46fW9e/cyceJE\nEhMTCQ8P58svvwSMKbcDBw4QFRVVdOz06dN5+eWXmTdvHgkJCbz55psAxMTEsHHjxmp7TxVWkA/r\n3oG1/4R3ehKz9Dbi6hf3jMkvUHzttmqtOiil+GLTUdM+PRoqh9B6EG3+XLPlEhdGVcnt++6oE0eu\ny6SS+79zdfl8o/n7ju7cmJpBPnBhWA4+l4iUUoOVUjEl/FoInBCRxgDO3y818T8CiFdKFRX2Ukqd\nUErlK6UKgPeAXlX5Xqpap06dWLZsGc8++yy//PILtWubKzi3bNmSrl27AtCjRw8OHToEwOnTpwkP\nDzcdO2DAAJRSzJw5k88++6xoys7hcBAYGMiFCxfwaft+NAqwOsnx7QzoHmM6ZP6mJHNnzWoQf+Qs\ne06YKyHf1NU+q5l8Qtc7zdvbv4Tcah7dnk+GPUtMu2r3vtu0/f2O46Rl5lZnVKRezPboxHpbbDMv\nR/sue6VNWATcC8xw/r7wEsfegdu0nIg0dpnaGwfsuNyAKnIPp7JER0cTHx/P4sWLmTp1KnFxcbz4\n4otFrwcFFd+sdDgcRVNzNWrU8KiRt337dlJSUoiIiCAszDxtlJ2dTXCwjz+PkDDXvN1xHKN6RvO/\ny46S55wX23vyIluOnqNbNdZ2+3SD+Wo1rl0D3Ym1vNoMhZB6kOG8z5adBjsXQefbqi+GI+uNhTmF\nbUUadCSmVxxNV60k6azx/yo7r4BvtiZz97Utqi2srxKOFX2+AVrVD6VHC/vVLvS5EVEpZgBDRGQv\nMNi5jYhEikjRCjgRCQWGAO4PHvxdRLaLyDZgEPCH6gm7aiQnJxMSEsLdd9/NlClTiI+PL9N5derU\nIT8/vygZpaSkcNddd7Fw4UJq1qzJkiXFV36pqanUq1ePgIByFKJM2gRb3G/NVaGLpzwbqHWfQL2a\nQVzf1txw7uP1VbZuxUNaZi7fbTevprqjt/XPNNmOI8Az6Wx8v3pjiBkPf0iEuBehThT0uA8/hx+3\nut2L/M+vR6pt1K2U8piWuy22mS0rvtgqESmlUpVScUqpNs4pvDPO/clKqZEux6UrpSKUUmlu509Q\nSnVSSnVWSt3kMjqype3bt9OrVy+6du3K9OnTmTp1apnPHTp0KKtXryYjI4Px48fz+uuv0759e154\n4QWmT59edNxPP/3EqFFlGPUpBVs/h3cHwftxsPgZyDpf+nmVIf4jo7JDoYg20Kw3AHe5/eD/dmsK\nqRezqQ7/jU/yeHZoQBvfvN/o83rcb94+ugFStlVvDGGNoP/TMDkBetwHwM09mpgW8CUmn2fz4bPV\nEs66/akeDfDGd7fn/UdbJSLNbNiwYWzbto0tW7awceNGYmONemoXL14kKiqKHTuKZx6feeYZpk2b\nVrQ9ceJE5syZQ0hICOvWrWPIkCGAca9o3briytWffvopjz76aNkCWvYCJDtHZTkXjaW2VS0vB351\nuzrucW/R8t6B0fVpXjek6KWc/AI+21j1N5XzCxQfrT1k2ndbbDMc+tmhiqkfbVS4drXxPSsiAT8/\n8DdanjStE0KcW5v3D93+3avK7DXm7zO0Q0MahNlz2lcnoqtU9+7dGTRoUKkPtI4dO5bo6OjSv6AI\nxD5o3vfru1VfqPK3hXDRpXROQCh0m1C06ecn3NPHPGc/b/1h8vKrNq4fd57gcGpxK+lAhx939Lbf\nTWSf0vNh8/a2+T7RRvy+vubmfUt2HCclLdPL0ZXjcGo6P+4yL1K4v5/vNBEsL52IrmIPPPBAqQ+0\n3nPPPWX/gj3uAz+Xe0ln9sPepRUPsDRKwYZZ5n1d74Qa5hWBt/ZoRnBA8Uc9OS3Lo0BkZftgtbmN\n9I1dIm17teozoodDbZdk7h8EJxKti8epX+sIWjeoWbSdX6CYu65q70XOWXvYVOy+Y2QtekbZb5FC\nIZ2ItMoT1tC4qevql9errj3EodWeDzf2/r3HYbVDAhjfvalp36yV+ykoqJq4tielseGg+Ur9wevs\ne7XqMxz+0OthCGsMQ1+GP+yAlv2r7vtdOAG7l5Q6qhcR7usbZdr3ybrDVbaU+0x6Dp9tPGLad3+/\nlrZcpFBIJyKtcvWZZN5O2mgUq6wKP//NvN1mGNRrXeKhj/Rvhevtmd0nLrBiV9XUn3vrx72m7T6t\nIugQaX33zitCr0fgia3Qd1JRc8Yqs+Yt+M/t8E5PY5VejvfCpuO7N6FuaGDR9oXsPOZW0b2iD1Yf\nIMOl3Xz9sCBu7GKvSgrudCLSKlfjzkZCcPXLPyr/+xxeC4d+Me8b8IzXw6PqhTLSrezJOyv3VfpS\n2x3H0jweMHx0YKtK/R5XtYAaxpRcVbtwHDZ9YPw5dR989zRs9f5IQkigv8eo94M1B0nP9lIwuILS\nMnKZs9Y87fdI/1YE+XufYrcDnYi0yueeEA6uggM/V+73WDnDvN1yIDS7dKGMx66/xrSdcOQcy347\n4eXoinlzuXk01K15OAOj9ZJt21nzFuS5PPQdFgld7/Z+PDChTwvCgotrBJzLyOXDNQcvcUb5fbDm\nIBezzV1+77rW/s+m6USkVb5mvSDKbe5+2QuVt4IuKw2y3UoODbxUEXZDx8jaDG5vXmr796W7K20F\n3cZDZzxGQ0/EtbH13P1V6exh2PiBeV//pyDg0otNagUHcL/bvaJZK/dz6kLlPLd2PC2L91YdMO17\n8LqWhATarUCOJ52ItKoR95J5O2UrbJ9f8rHlFVwbHvoRxr1rXKlG9YeofmU6dcqwdqZ7RftOXmT+\n5iTvJ5RRfoFi2iLzCq6uzfRoqFrkZcP6WbD6jcr5ej9MhXyX5BEWCd3Ltnr0wetamXphpefk8+by\nPZUS1mtLd5OZW3xvKCI00OPRBLvSiUirGs16Qsdx5n3Lp1VetQU/P+hyO0zeBOP+X5lPa9sojJvd\nVtD9fcmuy662sGDzURKTze/tzyPb69FQVSoogG1fGO3ElzwHK16Gk7su72seXGXUsXN1w/Nlvi9V\nOySAx29oY9r32caj/JZ8eZ/7bUnn+G+C+YLpqaHRhAWXo/SWD9OJ6CpXKZ1avYl7yfxc0YVkWPai\n9+MrIjAUajct/TgXTw2NJsi/+KN/NiOXv363s8IhnDyfxYzvzT8AR3duTK+WuhV4lcrLNPpPnXMu\nZS7IhW+eqPgUcE6GsSjBVWR36HJnycd7cfe1LWgRUVzNI79AMWXBVnIrOAWcnZfPlPnbTE9BRDes\nye02rLLtjf0nF33FtNqlH1OSxl3g0Spa3lwGldKp1Zu6LaHPRFjzpnOHGFeWSlnTYdOpce0aPB7X\nhteW7i7a91XCMUZ3bkxc+9J6LZoppfjjl9s4m1H8zEiQvx9/Gtm+0uLVvAgMheGvwgKXOnRH1xul\nf3qXsSyVq+UvGY33XI18zRh9lycsfz+mjurAw3M3Fe1LTD7Pv3/ezyS30VJZ/N+P+9h9wnxPjQT0\nOQAADe5JREFU9PlRHfB3XDnjiCvnnVyl5s6dS+fOnenSpQsTJhilbWbOnElMTAwxMTFFDe68dXOt\ntE6t3lz/HNS9BiJawwNLYMTfKpaEDq+r1HJBjwxoRbtG5udQnvpiK0fPZHg5o2SfrD/Myt3mDr5P\nDG5Dk/Aalx2jVgYdx3k2zlv6vFEBvjz2LjNKUrnqfg80ja1QWEM6NGS0W7vuN5fvZUM5u7iu25/K\nrJ/3m/aN797kirv3qEdENpaYmMhf//pX1q5dS7169Thz5gybN2/mww8/ZMOGDSil6N27NwMHDuTA\ngQNERkby3XffAZCWlua1U+uLL77IyZMnSUhIYNEiY768wp1aA2rAXfONluIBFfzhvHcZfHobtB4C\nN79nLFa4TAEOP/52c2fG/WtNURvvtMxcfv/JZhb8vi81Akt/LmP9gVSmf/ObaV9sizo8OuAaL2do\nlU4ERv4DDq2BHOeooSAXvrgHHlkJNRtc6mzDqT2wwK1OYp0oGPbKZYU2/aaOrNufSmp6DgB5BYrH\n5sWzaFI/mtYJKeVsOHomg0mfxpPvUgGkYa0gXhrd8bLi8kV6RGRjK1as4NZbb6VevXoA1K1bl9Wr\nVzNu3DhCQ0OpWbMm48eP55dffimxm2u1dWqNuKbiSejQavh8gtGQbO9SeO8G4wdHJejSLJxnhrU1\n7UtMPs/DczeRlev9nhlAwpGzPDxnk6kpWUigg5m3ddUVtqtbeDMY87Z53/ljMHcspJcyAkndDx+P\nNZrtFRI/GPfvy67cEFEziFfHdzLtO5Oewz2zf+XE+Ut3mE0+l8md768vSmKFZtzcmdohV8YCBVd6\nRFRZpqWVfoyFSurmOnnyZOs7tZ7cCTXqGL1eXClltJH49g/mpbSp+42b0/XLUBG8DB4beA1bj55j\naWLx8z+r953m9n+v41939yhxiu3bbclMmb/NtJQW4PVbu9A8ovQrXa0KdBwLSZNgnUtCOpkI798A\nt8+DRjEln5eb6VnBe/A0aH5tpYQ1tGMjnhzcxvSg84FT6Yz/11pm3d2dzk3DPc6JP3KWxz7ZzInz\n5pWcTw5uw6C2ZRjh2ZCtRkQicquIJIpIgYh4nbwVkeEisltE9onIcy7764rIMhHZ6/zdvuVqgRtu\nuIH58+eTmmpc9Z05c4b+/fvz9ddfk5GRQXp6Ol999RX9+/cvsZtrlXdqLU3GGZh3G7zdE356xXjW\n6Owh2PktzB0DC//HnIQAhs+ANoMrLQQR4R+3dqFDY3MtuK1JaQyZ+TOvLd3FtqRzHD2TwfLfTnDf\nh78y6dMEjyQ0ZVhbRnSyd70v2xs8zZi+dXX2ELw7EL59Co5s8DynUYx5NNX9Huj7eKWG9fgNbRjZ\nyXyhdexcJmPfWcNzX25jw4FUks5msP5AKlPmb+WWWWs9ktCYrpEey8KvJFJdbW0rg4i0BwqAfwPP\nKKU87kiKiAPYg9EqPAnYCNyhlPpNRP4OnFFKzXAmqDpKqVIfyY+NjVWbNpm/1c6dO2nf3vqVUXPm\nzOG1117D4XDQrVs3PvroI2bOnMns2bMBeOihh3jyySdZunQpU6ZMwc/Pj4CAAGbNmkVsbCwPPvgg\nd9xxB3379iUuLo6//OUvDBkyhFWrVvHss88WNclbsGAB69at4/XXXy8xjnL/fRTkwyc3w4Gfyn7O\n4GlwXdV0dz+TnsOd761n1/HyTz0+HteGp4ZUzghNu0y5mfCf38GBlZ6v+QXAi6dLPm/JnyHzDIx5\nB/wqv25bTl4BEz+Nr1BJqRExjfjnHd0IsOEqORHZrJQqdcWHrRJRIRFZifdE1AeYppQa5tz+E4BS\n6lUR2Q1cr5RKEZHGwEqlVFv3r+HOlxPR5YqPj+eNN97g448/vuRx48ePZ8aMGV6b5JX772PzR8Yz\nH2XhCIKb/s94gLUKnc/K5Yn/JPCT2yo4bwIcwl/GxHBHL/vX+rqi5OXA4qchfq55f2BN+POxks/J\nzzPuDZVzqXZ55OYX8MrinXzo1ln1Uh66riV/Gtnetvcdy5qI7JdiS9cEcO0FneTcB9BQKVXYEe04\n4PWhERF5REQ2icimU6fK9oPJjiq9U2tZdZtgTLMFlbICLqo/PLamypMQGLXCPri3Jy+Pi6F2jUtP\nQca2qMPCidfpJOSL/AONC5fbP4FaLg87OwK9n+Pwr9IkBMZKzZdu7Mj798SaHngtSVRECHMf6MXU\n0R1sm4TKw+dGRCKyHGhUwkvPK6UWOo9ZifcR0S3AcKXUQ87tCUBvpdQkETmnlAp3OfasUqrU+0RX\n8oioslT47yMrDbYvMJZon9ppXJmGRkCz3tDpNqNUkAXSs/NYuCWZn/ecZNfxC2TnFlAvLJDOTcMZ\n160JsS3q6PI9dpCfa7ST3/09XEiB+xdbHRFgTNWt2HWSxdtT2JlyngtZeYQF+9OpaW2GdmjIkA6N\nrogEVNYRkc+tmlNKXe6d6GOAa+2Lps59ACdEpLHL1FzVdEbTyi64NvR80PjlQ0KD/Lmzd3Pu7K1H\nPLbmCIBOtxi/fEigvx/DYxoxPKaka+6rz5U4NbcRaCMiLUUkEPgdUFjFcBFwr/PP9wILL+cb+dpo\n0ir670HTtMthq0QkIuNEJAnoA3wnIkud+yNFZDGAUioPmAQsBXYCXyilCuvzzwCGiMheYLBzu0KC\ng4NJTU296n8IK6VITU2t/OeLNE27avjcPSJfVNI9otzcXJKSkjweCL0aBQcH07Rp08p9xkjTNNuz\n7T0iuwgICKBly5alH6hpmqZdkq2m5jRN07Qrj05EmqZpmqV0ItI0TdMspRcrlIGInAIOV8GXrgd4\nKX5lC3aPH+z/HuweP9j/Pdg9fqi699BCKVVqFz+diCwkIpvKsqLEV9k9frD/e7B7/GD/92D3+MH6\n96Cn5jRN0zRL6USkaZqmWUonImu9a3UAl8nu8YP934Pd4wf7vwe7xw8Wvwd9j0jTNE2zlB4RaZqm\naZbSiUjTNE2zlE5EPkBEJovILhFJFJG/Wx1PRYjI0yKiRKSe1bGUl4i85vz73yYiX4lIeOlnWU9E\nhovIbhHZJyLPWR1PeYhIMxH5SUR+c37uy9g33reIiENEEkTkW6tjqQgRCReRBc7P/04R6WNFHDoR\nWUxEBgFjgC5KqY7APywOqdxEpBkwFDhidSwVtAyIUUp1BvYAf7I4nlKJiAN4BxgBdADuEJEO1kZV\nLnnA00qpDsC1wESbxV/oCYx2M3b1FrBEKdUO6IJF70UnIus9BsxQSmUDKKXs2DX2DeCPgC1Xviil\nfnD2sQJYj9HV19f1AvYppQ4opXKAzzAuaGxBKZWilIp3/vkCxg/AJtZGVT4i0hQYBbxvdSwVISK1\ngQHABwBKqRyl1DkrYtGJyHrRQH8R2SAiP4tIT6sDKg8RGQMcU0pttTqWSvIA8L3VQZRBE+Coy3YS\nNvtBXkhEooBuwAZrIym3NzEuwAqsDqSCWgKngA+d04vvi0ioFYHofkTVQESWAyU1p38e49+gLsb0\nRE/gCxFppXxoXX0p8f8ZY1rOp13qPSilFjqPeR5jymhedcZ2NRORmsCXwJNKqfNWx1NWIjIaOKmU\n2iwi11sdTwX5A92ByUqpDSLyFvAc8IIVgWhVTCk12NtrIvIY8F9n4vlVRAowChCeqq74SuMtfhHp\nhHFVtVVEwJjSiheRXkqp49UYYqku9W8AICL3AaOBOF+6CLiEY0Azl+2mzn22ISIBGElonlLqv1bH\nU079gJtEZCQQDNQSkU+UUndbHFd5JAFJSqnCkegCjERU7fTUnPW+BgYBiEg0EIhNKvkqpbYrpRoo\npaKUUlEYH+zuvpaESiMiwzGmWG5SSmVYHU8ZbQTaiEhLEQkEfgcssjimMhPjyuUDYKdSaqbV8ZSX\nUupPSqmmzs/974AVNktCOP+fHhWRts5dccBvVsSiR0TWmw3MFpEdQA5wr02uyK8kbwNBwDLnyG69\nUur31oZ0aUqpPBGZBCwFHMBspVSixWGVRz9gArBdRLY49/1ZKbXYwpiuRpOBec6LmQPA/VYEoUv8\naJqmaZbSU3OapmmapXQi0jRN0yylE5GmaZpmKZ2INE3TNEvpRKRpmqZZSiciTdM0zVI6EWmapmmW\n0olI0zRNs5RORJqmaZqldCLSNE3TLKUTkabZhIjc5mzHnisi13g5Zq7zmIMi0rC6Y9S0itCJSNPs\nYz6wFaNY8fPuL4rIXzAKiZ4BRiilTlRveJpWMbroqabZiLMh2zcYDfyilVIHnfvvx6jkng0MVkqt\nti5KTSsfPSLSNBtRSn0LrMdlVCQiQ4B3AQVM0ElIsxs9ItI0mxGRG4AfgVzgFuBjoBbwtB2bzGma\nTkSaZkMisgJnZ1+nfyqlnrAqHk27HHpqTtPs6W2XPy8C/mBVIJp2uXQi0jSbEZEI4FWXXQ6lVIFV\n8Wja5dKJSNNsRESCgYVANJAAFACjRKSPpYFp2mXQiUjTbEJEBGNhQj9gNzAY+ML58stWxaVpl0sv\nVtA0mxCRmRj3gk4CfZRSB0SkPbAD46IyTim1wsoYNa0i9IhI02xARB7HSEKZwI1KqQMASqmd6FGR\nZnN6RKRpPk5ExgELnJs3K6W+dnu9A7Ad48LyRudDr5pmG3pEpGk+TESuBeZh/F99yj0JASilfsOo\nQwfwv857SZpmG3pEpGmapllKj4g0TdM0S+lEpGmapllKJyJN0zTNUjoRaZqmaZbSiUjTNE2zlE5E\nmqZpmqV0ItI0TdMspRORpmmaZimdiDRN0zRL6USkaZqmWer/A6yxPKweImM6AAAAAElFTkSuQmCC\n", + "text/plain": [ + "<matplotlib.figure.Figure at 0x10c7c0dd8>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(x, ys, lw=4, ls='-', label=r'$\\sin(x)$') # linewidth = 4, linestyle = solid, raw string label\n", + "plt.plot(x, yc, lw=4, ls='--', label=r'$\\cos(x)$')\n", + "plt.legend() # show legend\n", + "plt.xlabel(r'$x$', fontsize=24) # label x axis\n", + "plt.ylabel(r'$y$', fontsize=24) # label y axis\n", + "plt.title('Sine and Cosine', fontsize=24)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that we used a few things:\n", + "1. We changed the line widths\n", + "2. We changed the line style\n", + "3. We labeled the plots\n", + "4. We changed the font size of the labels\n", + "\n", + "We also use a `raw string` because we're including Latex commands to render\n", + "mathematics. A `raw string` is preceded by the letter `r`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There is **much** more to plotting than this example, but this should get you\n", + "started. Some things you may want to look up are how to change the size of the\n", + "tick marks and tick labels and how to use a `config` file: [Customizing a\n", + "Plot](https://matplotlib.org/users/customizing.html). You may also want to\n", + "understand _contour_ plots as well as _scatter_ plots and other statistical\n", + "plots such as `pdfs` and `histograms`. Note that `seaborn` has fantastic\n", + "support for statistical plots." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,md" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/lecture06/notebooks/python_2.md b/lecture06/notebooks/python_2.md @@ -0,0 +1,415 @@ +--- +jupyter: + jupytext: + formats: ipynb,md + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 + language: python + name: python3 +--- + +# Basic Python + +* Booleans and Control Flow +* Functions +* Exceptions +* Plotting + + +We'll be embedding some HTML into our notebook. To do so, we need to import a +library: + +```python +from IPython.display import HTML +``` + +We'll also probably use `numpy` so we might as well import it too: + +```python +import numpy as np +``` + +## Booleans and Control Flow + +These are pretty standard things in any language. I'll just show you a little +bit of syntax here. However, I will put a slight emphasis on the exception +testing. It's important. + + +#### Testing for membership + +```python +some_primes = [2, 3, 5, 7, 13] +print(4 in some_primes) +print(13 in some_primes) +``` + +#### Some basic if/elif/else statements + +```python +x = 9 +if x in some_primes: + print('x is prime!') +elif x > 13: + print('x may or may not be prime.') +else: + print('x is definitely not prime.') +``` + +Breaking out of a loop: + +```python +for x in some_primes: + if x == 2: + print('Only even prime.') + break + +``` + +Continuing a loop. Notice that everything after the continue statement is +ignored. + +```python +i = 0 +while i < 10: + i += 1 + if i <= 5: + print(i) + continue + print(i-1) + else: + print('Done with this.') + break + +``` + +#### Some basic exception handling + +Python has a number of built-in exceptions ([Built-in +exceptions](https://docs.python.org/3/library/exceptions.html), [Python Standard +Exceptions](https://www.tutorialspoint.com/python/standard_exceptions.htm)). It +is usually a good idea to let Python raise exceptions for you since it's really +good at it. However, there are times when you may want to write your own +exception (we won't talk about that now) or when you want to press ahead even in +the face of an error. + +I can make that last statement clearer. You have two options: catch and +respond to errors when they're raised or ignore them. If you ignore them, then +Python's default exception-handling behavior takes over which will ultimately +print out the error message and terminate the program. If you respond to the +errors, then you need to tell your program what to do. In essence, you will +shift the control flow of your program if you choose this second option. + +Let's look at an example or two. + +```python +x, y = 1.0, 0.0 +z = x / y +z**2.0 +``` + +Python took care of error for us and terminated the program at the second line. + +But perhaps a division by zero isn't the end of the world. What if we have a +piece-wise function? + +```python +x, y = 1.0, 0.0 +try: + z = x / y +except ZeroDivisionError: + z = 0.0 +print('z = {}'.format(z)) +``` + +This could, of course, have been handled with an `if-else` block. + +One old motivation for using exception handling is to check the input arguments +of a function for validity. You can still do this, but the latest advice is to +just let Python's exception handler deal with it and terminate the program if +need be. + + +## Functions + +#### Python functions are first class: + +* You can assign variables to them +* You can pass them into functions +* You can return them from functions + + +### Example: + +```python +alist = [1,13,5,7] +newmax = max # Assign newmax to the function max +mymax1 = max(alist) # Get the maximum of the list using the max built-in +mymax2 = newmax(alist) # Get the maximum of the list using the new max function +print('Original maximum gives {0} and new maximum gives {1}.'.format(mymax1, mymax2)) +``` + +The syntax for defining functions is pretty straightforward. + +```python +def rect(w,h): + return w * h +def circle(r): + return np.pi * r * r +def parabola(x, a, b, c): + return a * x * x + b * x + c +``` + +Notice that the function name is preceded by the keyword `def`. The end of the +line **must** have a colon! The function body is indented. The quantity to be +returned is returned using the `return` statement. + +Because functions are first class, we can pass functions to other functions. + +```python +def parab_extrema(a, b, c, parab): + x_extreme = - b / 2.0 / a # Location of max or min + x_left = x_extreme - 1.0 # Point to the left of max or min + x_right = x_extreme + 1.0 # Point to the right of max or min + p_left = parab(x_left, a, b, c) # Value at left point + p_right = parab(x_right, a, b, c) # Value at right point + p_extreme = parab(x_extreme, a, b, c) # Value at max or min + # Check if extremum is maximum or minimum and print out result + if (p_left > p_extreme) & (p_right > p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme)) + elif (p_left < p_extreme) & (p_right < p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme)) + else: + print('Something went wrong.') +``` + +```python +a, b, c = 0.2, 1.0, -3.0 +parab_extrema(a, b, c, parabola) +a, b, c = -5.0, -5.0, 5.0 +parab_extrema(a, b, c, parabola) +``` + +There are some other convenient ways to interact with function arguments. + +One thing you may wish to do is provide a default argument to the function. If +that argument is not specified when the function is called, then the default +value is assumed. This is a type of keyword argument. + +Let's return to our nice parabola example. We'll make $b=0$ the default. + +```python +def parabola(x, a, c, b=0.0): + return a * x * x + b * x + c +``` + +Notice that we had to move our default argument to a position _after_ the +mandatory arguments. That hurts the readability a little bit in this example, +but we'll press forward regardless. + +Now call the `parab_extreme() function` again. + +```python +def parab_extrema(a, b, c, parab): + x_extreme = - b / 2.0 / a # Location of max or min + x_left = x_extreme - 1.0 # Point to the left of max or min + x_right = x_extreme + 1.0 # Point to the right of max or min + p_left = parab(x_left, a, c) # Value at left point + p_right = parab(x_right, a, c) # Value at right point + p_extreme = parab(x_extreme, a, c) # Value at max or min + # Check if extremum is maximum or minimum and print out result + if (p_left > p_extreme) & (p_right > p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme)) + elif (p_left < p_extreme) & (p_right < p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme)) + else: + print('Something went wrong.') +``` + +We changed the +[API](https://en.wikipedia.org/wiki/Application_programming_interface) a little +bit and so we had to update the calls to `parab()`. However, everything works +just fine if we're careful. + + +It's probably better to give all the parameter arguments default values. Let's +re-write the API again. + +```python +def parabola(x, a=1.0, b=-1.0, c=-1.0): + return a * x * x + b * x + c + +def parab_extrema(parab, a=1.0, b=-1.0, c=-1.0): + x_extreme = - b / 2.0 / a # Location of max or min + x_left = x_extreme - 1.0 # Point to the left of max or min + x_right = x_extreme + 1.0 # Point to the right of max or min + p_left = parab(x_left, a, b, c) # Value at left point + p_right = parab(x_right, a, b, c) # Value at right point + p_extreme = parab(x_extreme, a, b, c) # Value at max or min + # Check if extremum is maximum or minimum and print out result + if (p_left > p_extreme) & (p_right > p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a minimum.'.format(x=x_extreme, y=p_extreme)) + elif (p_left < p_extreme) & (p_right < p_extreme): + print('The extremum for this parabola has coordinates ({x:4.3f}, {y:4.3f}) and is a maximum.'.format(x=x_extreme, y=p_extreme)) + else: + print('Something went wrong.') + +parab_extrema(parabola) +``` + +Great! Looks pretty nice. + +But there's more! We can also provide _positional_ and _keyword_ arguments to a +function. This allows permits a variable number of arguments to be passed to a +function. + +* positional arguments: `def func(*args):` + + Python collects all the remaining positional arguments into a tuple. You + can then access the tuple with the usual indexing. +* keyword arguments: `def func(**kwargs):` + + Python collects all the remaining keyword arguments into a dictionary. You + can then access the dictionary with the usual indexing. + + +### Variable Positional Arguments + +We will once again work with the quadratic equation example. This time, we'll +just work with the `parabola` function to save some space. Let's change the +`parabola` function to permit a variable number of arguments. + +```python +def parabola(x, *args): + return args[0] * x * x + args[1] * x + args[2] +parabola(1.0, 1.0, -1.0, -1.0) +``` + +Seems to work okay. But this is not a very robust code. Everything breaks if +we don't provide the exact number of necessary arguments. + +```python +parabola(1.0) +``` + +### Variable keyword arguments + +We can make our API more flexible. Let's give more descriptive names to the +coefficients $a$, $b$, and $c$. We'll call $a$ the `width` since it controls +the width of the parabola, $b$ `trans` since it controls the horizontal +translation of the parabola, and we'll call $c$ `shift` since it controls the +vertical shift of the parabola. Our `parabola` function might now look like: + +```python +def parabola(x, **kwargs): + print(kwargs) + return kwargs['width'] * x * x + kwargs['trans'] * x + kwargs['shift'] +``` + +Calling it gives: + +```python +parabola(1.0, width=1.0, trans=-1.0, shift=-1.0) +``` + +**Note:** Using variable positional and keyword arguments provides exceptional +flexibility in how you design your programs. + + +One final note about variable arguments: You can perform the reverse operation +by passing in the `*` or `**` operators to the function. This will _unpack_ the +arguments whereas the previous pattern _packed_ the arguments. Let's take a +quick look. + +```python +def parabola(x, a, b, c): + return a * x * x + b * x + c +``` + +```python +# Store coefficients in a list +coeffs = [1.0, -1.0, -1.0] +parabola(1.0, *coeffs) + +# Store coefficients in a dictionary +coeffs = {'a':1.0, 'b':-1.0, 'c':-1.0} +parabola(1.0, **coeffs) +``` + +--- + +## Plotting + +There are many, many ways to make plots in Python. The most common way is to +use [`matplotlib`](https://matplotlib.org/). + +Another package, which is gaining popularity, is called +[`seaborn`](https://seaborn.pydata.org/). + +I don't care which package you use, as long as your plots are readable and +reproducible. + + +To make plots in the Jupyter notebook, you need to include the line `%matplotlib +inline` before you make any plots. This line ensures that the plots will be +displayed in your notebook and not in a separate window. + +```python +%matplotlib inline +``` + +Next, you should `import matplotlib`: + +```python +import matplotlib # Import all of matplotlib +import matplotlib.pyplot as plt # Only import pyplot (which includes the plot function) and give it the alias `plt` +``` + +Now you're basically ready to do some plots. + +**WARNING!** When making plots in an actual Python script, you must **always** +include the command `plt.show()` at the **end** of your program. **Always.** If +you don't do so, then your plots will not display and you will be wondering +where they are. However, when plotting in the Jupyter notebook, there is no +need to use `plt.show()`. + + +We can generate some toy data using `numpy`. + +```python +x = np.linspace(-2.0*np.pi, 2.0*np.pi, 500) # x-grid +ys = np.sin(x) # sin function +yc = np.cos(x) # cos function +``` + +Now plot! + +```python +plt.plot(x, ys, lw=4, ls='-', label=r'$\sin(x)$') # linewidth = 4, linestyle = solid, raw string label +plt.plot(x, yc, lw=4, ls='--', label=r'$\cos(x)$') +plt.legend() # show legend +plt.xlabel(r'$x$', fontsize=24) # label x axis +plt.ylabel(r'$y$', fontsize=24) # label y axis +plt.title('Sine and Cosine', fontsize=24) +``` + +Notice that we used a few things: +1. We changed the line widths +2. We changed the line style +3. We labeled the plots +4. We changed the font size of the labels + +We also use a `raw string` because we're including Latex commands to render +mathematics. A `raw string` is preceded by the letter `r`. + + +There is **much** more to plotting than this example, but this should get you +started. Some things you may want to look up are how to change the size of the +tick marks and tick labels and how to use a `config` file: [Customizing a +Plot](https://matplotlib.org/users/customizing.html). You may also want to +understand _contour_ plots as well as _scatter_ plots and other statistical +plots such as `pdfs` and `histograms`. Note that `seaborn` has fantastic +support for statistical plots. diff --git a/lecture07/examples_in_slides/01.py b/lecture07/examples_in_slides/01.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +def four(): + return 0x4 + +t = (1, 2.0, '3', four) +for item in t: + print(type(item)) diff --git a/lecture07/examples_in_slides/02.py b/lecture07/examples_in_slides/02.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +def Complex(r, i): + """ + Construct a complex number + Arguments: + r: real part + i: imaginary part + """ + return (r, i) + + +# interface with complex numbers +def real(c): + """Get the real part of a complex number c""" + return c[0] + + +def imag(c): + """Get the imaginary part of a complex number c""" + return c[1] + + +def string(c): + """Represent a complex number c as a string""" + return f"{c[0]:e} + i{c[1]:e}" + + +z = Complex(1, 2) +print(real(z), imag(z), string(z)) diff --git a/lecture07/examples_in_slides/03.py b/lecture07/examples_in_slides/03.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +def Complex(r, i): + """ + Construct a complex number + Arguments: + r: real part + i: imaginary part + """ + def implementation(method): + if method.lower() == 'real': + return r + elif method.lower() == 'imag': + return i + elif method.lower() == 'string': + return f"{r:e} + i{i:e}" + + return implementation + + +z = Complex(1, 2) +print(z('real'), z('imag'), z('string')) diff --git a/lecture07/examples_in_slides/04.py b/lecture07/examples_in_slides/04.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +def Complex(r, i): + """ + Construct a complex number + Arguments: + r: real part + i: imaginary part + """ + def implementation(method, z=None): + nonlocal r, i + if method.lower() == 'set_z': + assert z is not None + r, i = z + elif method.lower() == 'real': + return r + elif method.lower() == 'imag': + return i + elif method.lower() == 'string': + return f"{r:e} + i{i:e}" + + return implementation + + +z = Complex(1, 2) +z('set_z', (3, 4)) +print(z('real'), z('imag'), z('string')) diff --git a/lecture07/examples_in_slides/05.py b/lecture07/examples_in_slides/05.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +class Complex(): + """Complex number type""" + def __init__(self, real, imag): + self.real = real # real part + self.imag = imag # imaginary part + + +z = Complex(1, 2) +print(z.__dict__) +print(vars(z)) +print(Complex) +print(type(z)) +print(z.real) +print(z.imag) diff --git a/lecture07/examples_in_slides/06.py b/lecture07/examples_in_slides/06.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +class Animal(): + """Base class for animals""" + def __init__(self, name, age): + self.name = name + self.age = age + + # class method, note how `self` is passed as argument + def speak(self): + """Sounds animals can make""" + raise NotImplementedError + + +class Dog(Animal): # Dog is a derived class, it inherits from Animal + """Dog is a derived Animal class""" + def speak(self): + """Special sounds dogs make""" + return "Woof" + + def my_id(self): + print(id(self)) + + +class Cat(Animal): # Cat is a derived class, it inherits from Animal + """Cat is a derived Animal class""" + def __init__(self, name, age): + self.name = f"A very special cat: {name}" # cats have a special name string + + def speak(self): + """Special sounds cats make""" + return "Meow" + + +dog = Dog("Snoopy", 6) +cat = Cat("Kitty", 4) +print(dog.speak()) +print(cat.speak()) + +if isinstance(dog, Animal): + print(f"dog ({id(dog)}) is an instance of Animal") +if isinstance(cat, Animal): + print(f"cat ({id(cat)}) is an instance of Animal") diff --git a/lecture07/examples_in_slides/07.py b/lecture07/examples_in_slides/07.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +class Base(): + """Base class""" + def __init__(self, a): + self.a = a # some data required in the base class + + def explain(self): + print(f"Executing from base class: data=`{self.a}`") + + +class Derived(Base): + """Derived class""" + def __init__(self, a, b): + super().__init__(a) # properly initialize the base class + self.b = b # some data specific to the derived class + + def explain(self): + # 1. Call the base class method first + super().explain() + # 2. Then do special work required for the derived class + print(f"Executing from derived class: data=`{self.b}`") + + +a = "base class data" +b = "derived class data" +base = Base(a) +derived = Derived(a, b) + +base.explain() +derived.explain() +print(Base.__mro__) +print(Derived.__mro__) diff --git a/lecture07/yapf/.style.yapf b/lecture07/yapf/.style.yapf @@ -0,0 +1,2 @@ +[style] +based_on_style = pep8 diff --git a/lecture07/yapf/README.md b/lecture07/yapf/README.md @@ -0,0 +1,15 @@ +# Demo for code formatting with yapf + +To format code, run `./maintenance/format_code.sh` from the project root. + +``` +yapf_demo +├── file1.py +├── .git +├── maintenance +│  └── format_code.sh +├── other_code +│  └── file2.py +├── README.md +└── .style.yapf +``` diff --git a/lecture07/yapf/file1.py b/lecture07/yapf/file1.py @@ -0,0 +1,14 @@ +x = { 'a':37,'b':42, + +'c':927} + +y = 'hello ''world' +z = 'hello '+'world' +a = 'hello {}'.format('world') +class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y +def f ( a ) : + return 37+-+a[42-x : y**3] diff --git a/lecture07/yapf/maintenance/format_code.sh b/lecture07/yapf/maintenance/format_code.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# File : format_code.sh +# Created : Sat Sep 11 2021 11:51:15 AM (-0400) +# Author : Fabian Wermelinger +# Description: Maintenance script to format python code. Formatting rules are +# defined in .style.yapf at the project root. +# Copyright 2021 Harvard University. All Rights Reserved. + +yapf --in-place --recursive --parallel . diff --git a/lecture07/yapf/other_code/file2.py b/lecture07/yapf/other_code/file2.py @@ -0,0 +1,14 @@ +x = { 'a':37,'b':42, + +'c':927} + +y = 'hello ''world' +z = 'hello '+'world' +a = 'hello {}'.format('world') +class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y +def f ( a ) : + return 37+-+a[42-x : y**3] diff --git a/lecture08/FrenchDeck.py b/lecture08/FrenchDeck.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# File : FrenchDeck.py +# Created : Fri Sep 17 2021 08:12:06 PM (-0400) +# Description: Code based on Example 1-1 Fluent Python: Clear, Concise, and +# Effective Programming by Luciano Ramalho (O'Reilly Media, 2015) +# Copyright 2021 Harvard University. All Rights Reserved. +from collections import namedtuple + +Card = namedtuple('Card', ['rank', 'suit']) + + +class FrenchDeck: + """French deck of 52 playing cards""" + ranks = [str(rank) for rank in range(2, 11)] + list('JQKA') + suits = 'spades diamonds clubs hearts'.split() + + def __init__(self): + """Initialize ordered deck of cards""" + self._cards = [ + Card(rank, suit) for suit in self.suits for rank in self.ranks + ] + + def __len__(self): + """Return length of deck""" + return len(self._cards) + + def __getitem__(self, index): + """Return card at index""" + return self._cards[index] + + def __setitem__(self, index, card): + """Set a card at index""" + self._cards[index] = card # this method does not return a value + + def __str__(self): + """Pretty print card deck""" + map_utf8 = { + 'clubs': '♣', + 'diamonds': '♦', + 'hearts': '♥', + 'spades': '♠' + } + cpl = 13 # number of cards per printed line + pretty = [] + for line in range((len(self._cards) + cpl - 1) // cpl): + for card in self._cards[line * cpl:(line + 1) * cpl]: + pretty.append(f" {card.rank:>2}{map_utf8[card.suit]}") + pretty.append('\n') + return ''.join(pretty).rstrip() diff --git a/lecture08/Thing.py b/lecture08/Thing.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# File : Thing.py +# Description: `Thing` class to study basic OOP design +# Copyright 2022 Harvard University. All Rights Reserved. + +class Thing: + """Simple class for a 'thing'""" + def __init__(self, thing): + self.state = thing + + def __str__(self): + return f"{self.state}" + + def __add__(self, other): + """This method implements addition '+'""" + print(self.state) + return Thing(f"{str(self)} + {str(other)}") + + def __iadd__(self, other): + """This method implements augmented addition assignment '+='""" + print(self.state) + self.state = f"{str(self)} + {str(other)}" + return self + + +def mutate(x): + y = Thing('B') + x += y # calls __iadd__: study line 23 + return x + + +def rebind(x): + y = Thing('C') + x = x + y # calls __add__: study line 17 + return x + + +if __name__ == "__main__": + A = Thing('A') + B = mutate(A) # after this function call A and B are the same object + C = rebind(A) # after this function call A and C are two different objects + print(f"id(A) = {id(A)}") + print(f"id(B) = {id(B)}") + print(f"id(C) = {id(C)}") diff --git a/lecture09/python_project/.gitignore b/lecture09/python_project/.gitignore @@ -0,0 +1,4 @@ +/dist +*.pyc +*.egg-info +**/__pycache__ diff --git a/lecture09/python_project/LICENSE b/lecture09/python_project/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Harvard University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lecture09/python_project/README.md b/lecture09/python_project/README.md @@ -0,0 +1,18 @@ +# A simple Python project for CS107/AC207 + +Demonstration of PEP517 and PEP518 for package building and distribution. + +> See [https://packaging.python.org/tutorials/packaging-projects/](https://packaging.python.org/tutorials/packaging-projects/) + +## Steps + +1. Clean previous distributions: `rm dist/*` +2. Build the package release: `python -m build .` +3. Upload to [test.pypi.org](test.pypi.org): `twine upload --repository testpypi dist/*` +4. Repo: [https://test.pypi.org/project/Fall2022-CS107/](https://test.pypi.org/project/Fall2022-CS107/) + +## Installing the package + +```bash +python -m pip install -i https://test.pypi.org/simple/ Fall2022_CS107 +``` diff --git a/lecture09/python_project/pyproject.toml b/lecture09/python_project/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" # use setuptools for building diff --git a/lecture09/python_project/setup.cfg b/lecture09/python_project/setup.cfg @@ -0,0 +1,36 @@ +[metadata] +; The name is normally the name of your package. In this test project it would +; be cs107_package (which is what we have in ./src). It can be different though +; as in this example. The name you specify here is what one would have to type +; using `python -m pip install <name>`, while the package name(s) in ./src is +; what one must type using the `import <package name>` statement. Note that +; PyPI is following PEP8 and converts underscores `_` in the name to hyphens +; `-`. You can still use `python -m pip install Fall2022_CS107` however. +name = Fall2022_CS107 +version = 0.0.2 +author = Fabian Wermelinger +author_email = fabianw@seas.harvard.edu +description = CS107/AC207 test package +long_description = file: README.md +long_description_content_type = text/markdown +url = https://code.harvard.edu/CS107/main/tree/master/lecture/code/lecture09/python_project +classifiers = + Intended Audience :: Developers + Programming Language :: Python :: 3 + Topic :: Software Development :: Libraries :: Python Modules + +[options] +; for our test package we chose to put the source code into `src` +package_dir = + =src + +; required to automatically find our package in `src` +packages = find: + +; additional package dependencies (examples): this package requires numpy +install_requires = + numpy + +[options.packages.find] +; additional config required for find above +where = src diff --git a/lecture09/python_project/src/cs107_package/__init__.py b/lecture09/python_project/src/cs107_package/__init__.py @@ -0,0 +1,9 @@ +import importlib.metadata +from .subpkg_1 import (foo, bar) +from .subpkg_2 import baz + +# The following will get the version of your project from the `setup.cfg` +# configuration: +__version__ = importlib.metadata.version('Fall2022_CS107') + +__all__ = ['foo', 'bar', 'baz'] diff --git a/lecture09/python_project/src/cs107_package/__main__.py b/lecture09/python_project/src/cs107_package/__main__.py @@ -0,0 +1,3 @@ +import datetime as dt + +print(f"Hello from cs107_package! Today is: {dt.datetime.now()}") diff --git a/lecture09/python_project/src/cs107_package/subpkg_1/__init__.py b/lecture09/python_project/src/cs107_package/subpkg_1/__init__.py @@ -0,0 +1,4 @@ +from .module_1 import foo +from .module_2 import bar + +__all__ = ['foo', 'bar'] diff --git a/lecture09/python_project/src/cs107_package/subpkg_1/module_1.py b/lecture09/python_project/src/cs107_package/subpkg_1/module_1.py @@ -0,0 +1,5 @@ +class Foo: + pass + +def foo(): + print("cs107_package.subpkg_1.module_1.foo()") diff --git a/lecture09/python_project/src/cs107_package/subpkg_1/module_2.py b/lecture09/python_project/src/cs107_package/subpkg_1/module_2.py @@ -0,0 +1,6 @@ +from ..subpkg_2 import module_3 as mod3 + + +def bar(): + mod3.baz() + print("cs107_package.subpkg_1.module_2.bar()") diff --git a/lecture09/python_project/src/cs107_package/subpkg_2/__init__.py b/lecture09/python_project/src/cs107_package/subpkg_2/__init__.py @@ -0,0 +1,3 @@ +from .module_3 import baz + +__all__ = ['baz'] diff --git a/lecture09/python_project/src/cs107_package/subpkg_2/module_3.py b/lecture09/python_project/src/cs107_package/subpkg_2/module_3.py @@ -0,0 +1,2 @@ +def baz(): + print("cs107_package.subpkg_2.module_3.baz()") diff --git a/lecture09/python_project/src/cs107_package/subpkg_2/module_4.py b/lecture09/python_project/src/cs107_package/subpkg_2/module_4.py @@ -0,0 +1 @@ +pass diff --git a/lecture09/python_project/src/cs107_package/subpkg_2/module_5.py b/lecture09/python_project/src/cs107_package/subpkg_2/module_5.py @@ -0,0 +1 @@ +pass diff --git a/lecture09/pythonpath/README.md b/lecture09/pythonpath/README.md @@ -0,0 +1,22 @@ +# Setting up `PYTHONPATH` to point into custom Python code + +You can use the `PYTHONPATH` environment variable to let the Python interpreter +know where to look for packages. This is useful when you develop in order to +avoid installation cycles of your packages if you only want to test. + +We have a CS107/AC207 test project called `cs107_package` in the parent +directory of this current directory. When we are in the Python interpreter a +statement like `import cs107_package as cp` would fail because its location is +not in the search path of the interpreter. + +You can solve this by adding the path to the Python package into the +`PYTHONPATH` environment variable. See the `setup_pythonpath.sh` shell script +in this directory. You can simply source it to add the new path. + +```bash +source setup_pythonpath.sh +``` + +**NOTE:** you should avoid adding relative paths to environment variables like +`PATH` or `PYTHONPATH` as it will break you setup. It is done here because it +will be easier for you to just test this out of the box. diff --git a/lecture09/pythonpath/setup_pythonpath.sh b/lecture09/pythonpath/setup_pythonpath.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +export PYTHONPATH=../python_project/src:$PYTHONPATH diff --git a/lecture11/newtons_method/newton.py b/lecture11/newtons_method/newton.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# File : newton.py +# Description: Newton's method with exact Jacobian representation +# Copyright 2021 Harvard University. All Rights Reserved. +import numpy as np + +f = lambda x: x - np.exp(-2.0 * np.sin(4.0 * x) * np.sin(4.0 * x)) +J = lambda x: 1.0 + 16.0 * np.exp(-2.0 * np.sin(4.0 * x)**2 + ) * np.sin(4.0 * x) * np.cos(4.0 * x) + + +def newton(f, J, x_k, tol=1.0e-8, max_it=100): + root = None + for k in range(max_it): + dx_k = -f(x_k) / J(x_k) + if abs(dx_k) < tol: + root = x_k + dx_k + print(f"Found root {root:e} at iteration {k+1}") + break + print(f"Iteration {k+1}: Delta x = {dx_k:e}") + x_k += dx_k + return root + + +if __name__ == "__main__": + import argparse + + def parse_args(): + # yapf: disable + parser = argparse.ArgumentParser(description="Newton-Raphson Method") + parser.add_argument('-g', '--initial_guess', type=float, help="Initial guess", required=True) + parser.add_argument('-t', '--tolerance', type=float, default=1.0e-8, help="Convergence tolerance") + parser.add_argument('-i', '--maximum_iterations', type=int, default=100, help="Maximum iterations") + # yapf: enable + return parser.parse_args() + + args = parse_args() + newton(f, J, args.initial_guess, args.tolerance, args.maximum_iterations) diff --git a/lecture11/newtons_method/newton_fd.py b/lecture11/newtons_method/newton_fd.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# File : newton_fd.py +# Description: Newton's method with approximate Jacobian representation +# Copyright 2021 Harvard University. All Rights Reserved. +import numpy as np + +f = lambda x: x - np.exp(-2.0 * np.sin(4.0 * x) * np.sin(4.0 * x)) +J = lambda x, eps: ( + f(x + eps) - f(x) +) / eps # Finite-Difference approximation of J + + +def newton(f, J, x_k, tol=1.0e-8, max_it=100, eps=1.0e-8): + root = None + for k in range(max_it): + dx_k = -f(x_k) / J(x_k, eps) + if abs(dx_k) < tol: + root = x_k + dx_k + print(f"Found root {root:e} at iteration {k+1}") + print(f(root)) + break + print(f"Iteration {k+1}: Delta x = {dx_k:e}") + x_k += dx_k + return root + + +if __name__ == "__main__": + import argparse + + def parse_args(): + # yapf: disable + parser = argparse.ArgumentParser(description="Newton-Raphson Method") + parser.add_argument('-g', '--initial_guess', type=float, help="Initial guess", required=True) + parser.add_argument('-t', '--tolerance', type=float, default=1.0e-8, help="Convergence tolerance") + parser.add_argument('-i', '--maximum_iterations', type=int, default=100, help="Maximum iterations") + # yapf: enable + return parser.parse_args() + return parser.parse_args() + + args = parse_args() + newton(f, J, args.initial_guess, args.tolerance, args.maximum_iterations) diff --git a/lecture11/sympy/jacobian.ipynb b/lecture11/sympy/jacobian.ipynb @@ -0,0 +1,127 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ec28b876-491e-4a36-9f7a-9741091710a8", + "metadata": {}, + "outputs": [], + "source": [ + "# File : jacobian.ipynb\n", + "# Description: Compute the Jacobian of f(x) symbolically\n", + "# Copyright 2021 Harvard University. All Rights Reserved.\n", + "import sympy as sym" + ] + }, + { + "cell_type": "markdown", + "id": "ca02b856-bad2-4321-bc40-1b241e178564", + "metadata": {}, + "source": [ + "Initialize the printing enginge:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "43378bb3-7fb1-4353-b25d-a244aabbc1a8", + "metadata": {}, + "outputs": [], + "source": [ + "sym.init_printing()" + ] + }, + { + "cell_type": "markdown", + "id": "7a55d860-a790-42c6-9139-86bc8216f219", + "metadata": {}, + "source": [ + "Define the symbolic variable $x$" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b45dfea4-084e-45c5-bbf8-35c1d783151b", + "metadata": {}, + "outputs": [], + "source": [ + "x = sym.symbols('x')" + ] + }, + { + "cell_type": "markdown", + "id": "e2ee0254-3bc6-4b0c-8dc4-cf709cb51c51", + "metadata": {}, + "source": [ + "Define the function $f(x)$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0d6e1538-8be1-4dfb-b277-2ffe5ba50510", + "metadata": {}, + "outputs": [], + "source": [ + "f = x - sym.exp(-2 * sym.sin(4 * x)**2)" + ] + }, + { + "cell_type": "markdown", + "id": "39954478-feda-44b6-898e-9db0d170123a", + "metadata": {}, + "source": [ + "Compute the derivative $\\mathrm{d}f/\\mathrm{d}x$ (Jacobian)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e705c9bb-d927-428b-8273-574282294e24", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASYAAAAbCAYAAADRVyIdAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAKwUlEQVR4Ae2c7bXcNBCGN/ekAEg6CB1AqIDQAR8VBDqAk3/5x4EOAhVA6AA6SEgH0AHhdhCeR/EI2Za9tne9H8Fzjq7kkTQajUevRvImd968ebPb6Dot8PTp00do/h7pAekD0nN4v5NvtFngqi1w96q135QXiN7XDOQC1D+kOz5vtFngmi1wc83Kb7rvPipsYNR0WzwPFgGxz0i2Pwoh65ujCNqEbBZoLHBnO8ot94UmSnnSSIiF/hj+JICIkWn/FeVvyT2OLSL6/kbH78lHj3LNWH8NtYP/B6kEvEn60OcZ6etJjbdGmwX2WGCLmPYYaE+1QCCgmD6n7WvSH3v61KpfwnxWq5jCY2z7qsM+UPK49+lQO/hGPh9OGbPSxmPl9xX+xtosMNsCGzDNNlmrw1csRi+gg1yYD+DNWty0f0X6IYTMyennmIKSMvYdqWxbBUD6Oo+/5oxdtqW/oPiIXPDbaLPAQRbYLr8PMt/Oo4vRzl5qQMOFf4/kke1PeD82C/k5zy7qdHFd8B7ANxIzN31Mnc+JKBudCYLfUJan/DGAc4zecQueYKKcH0k9oj4AT70Ft89IqUxdCWYJnKj7lXTxhO5uIqX+R9f5FGMcXelC4Cn0r41xU+iwFWdaAIMKLOV9kove+5tXpSiejUbuk/9KcvEbtaTIgmf7t8Ci4CVAavoJOC4kQSER5Y9Id4o0eEdFG2WVujZSUmbkVwU0+IKSelsvqHqXZPkhSTAr6QUPn5aMSy038+rqv4a6vrMA9jXkrybznDZqRUwo4oty99bhh5x4NUN0BU/Vh3bu4iX9DK8FDmXlGuVGV0GjdnHs3ZNRjWDkJfXvlEsgsH6Iynsj2wkwS8h+vXHQQ53HIpxb2kRUEfo7vpf8rywUpM8s1S+LQa4yjAY9olajuNx4QQGZztmNonwHPUnUux6+Jm9tHL2GIwz6+q4/JAn+R5/LyNAHVaHrWW10FwV0tp9IOu1D0sGOhYzFNEcf2qqrQJovfuH5bBqMHqjrEf10wi7A9drB0FFjoab6Rg/7VgGdeu9/jCS+JanbzmdSCTqye0QbF3tQWQ7e1Nz33OqPbO13j7w1n1IgdeViMvJLixR+F5Ts1gO+UtaMsnqpbw3kZ4jpN0Vv5T4hnyLbd1WbZ1/wCIexfiD5tfMXUusdjHQ7WxU6nt1GApOGSvcWlA05Dw5vkaMDG8KWTj3J0DP10XGMjsoFrlEHF9qQEsjQAWcfQ+jnIhIYU9/meUeedaCsTV+SRxt/HpCAjPxUpD7apiSfP0CvAORU3zy/IM+RFGXnkKMnnmv3M9oiz7scaE4Z2UYZ75Nu5/Sb2Na5Vj8AlP0Z+9jHL8d07MXRV6nfyuWz2+hmpQnq4MnJV5K/w3EMNV0sLfCDbyQyG2CW6Mk4LkQdznsXw3X1MSrqRg62i8W/o506d9vA2kuLbcqYAq96ZJJHElRToiJ0/A6e90qCj78ml74klaDjPLuk/D+7zCXPjLsGKKnKF8hu+UxXP+oTCMM/mg7NmI69+B129Vzx+ew2ullxcmuLdudxBz+a8yxQ2HsQo0PzSN4lVHWC7z2T9XnnpKyjepTeUU47ecNLIFHwjLI8an8JzzGXkBfzLXAKIfAjipPl77P8dbhA5PHD6CHuxpyDzzVgckPIURbliyLnhEIluA7pp41HwWuo4x6+Y3+xp81Zqy/FRnfPaoXDBneRutDc3dzN/yZ5r1T9h6y0EwCeNO3IUlsjhSqI2GAf0Tf9O7UJ7Vys1QXbjJ9/AqCsAZ4L5dDFEoDYAxXGrMqHXx49yiOzqmainfbdke9d+LTxnQmu2j71IxcMvKMTOD2im3uki2sG+wjg8n8hOReBRvqY5NFz9DKbNgLn4Byo2yFD0E0bhM81oo06C+SSYz8mqZd+KLWOwW9Z6a9jq8Poe2zkOz/tE9SSWbSJCLXn+7QZtHMIreQXYaNrBqZw6Ie8gLzQKP9D8otRBgLKLgKd7XPKHml25L40gSr3lf8uE3NOC528dj906NQD9EblMLbvzYjMBZCJ57hUF9gEKCPQTDz73uS7EO+RPDonICL3XXq57PFzDBjdzAZBh76CS75HozxE6h/6CmQCpsDhRqd8bZH9j3KQuuuLg9To4NyzD8Ozz3Py9FMUcucreHttkedL+TfrSF62j9qZvkN0ETa6GdLukvmN0VXRl9DdfdxNf4o25DqbxxCdKYESZcmXLf9/RdjABeWXxQD2g+ePLKOHtHAmCNPx3Uy643cB4/WALBfiI/rnhU853uvooqefYw7JdTjt0vUn+ZmoL4+88m9JRm7RT9AUmGrk2PrjGAk4AlCeH89hK8eSbFMDYTdZ/VzgmmpnmrboImx0t6XSzAcMoDPVnMGXs6O+PAaEdC9cW0eXqFiQ592i6Otuo/P4Ygydw+HfY1x3N+k+yd1lNKxPLd/BP8zbnT2c/RgznPwZXJuTXKBGttrfzcFFtu8YVur5snwoyvvmpF/G4i66JV/VZ8JXWnWdh5foWvqdPzvQp5Nc8jHftt+gjvS1TlBp2QK+IJWAirL1gtsLUouoUw95Hot9x0vsfBE2OhSYasCzwyDuIB4XWgbWYscg5BpuK6rqZM0YsTMJnHMdvxHx7mba8FizWyDLxfyEpJ8YXbjL+yv6qj9Rvyoxrr7ixlUCTnVM2rzqVHiZ/V2HN/Q4uOibDuGz3pcOUbQZe3+Cl3Q0O5/aRgcB09u5n+2vu228pJoSpZP1dpdah423vgUaB39N7rEj3e9RTtEKuT+76C78YyplBGFU0iX9yH+H6BGppBSdNHyPV637SJ7d9JSXj13wknzyGnBYpw5DFD479uPgaJPGGRCkrs5piZ0vwkbXDEyG3V1H8j25SxhRxTHNF3nfii7Rxk/i2am69dvzKhZIix3JOZrmHRgteRnu8XtNYNIXepsZY+sr4S8U3xJ8f8NVfhkUDPQ5IzzbG+HpawEWPKZflbcATGZDRkxl2+CnHDnKcv7aoUfUJX+1HZXaq+W78AVKSR2X2vkibHSTpvHfn1jAGvASaFAfXoIvRafJF42UdRxD68eF8tbHC0ts25EEtjUXQaHCVuxYwH8S0t3xfe6CQ7eNYg7xTd+3n/enkuOXOuhHJiMR+a9JmeBZ9yIz+gU3zX0+5x2V1yBGkZl41o+j7yeU/RGk4FOSbfwiF3acaudSxkXYKP0PlkwkIg8Nq8FVTuT0gji+NvA4jehz0B3THH1o68sI59Fp/dVyvMCkMM9eegty8ZsPgSnv2KnR9uckFsDu+pi7fXmP4rtJHyOod7F5/2Q7yUXmRmOkE/dStw3fux35Ri62T3xkVC+gG9l+PRw7Ku2od9NSbujgJijguBb0t+RHtBME9HXn40cXAasVxcDLRJ39/PIXwJHrygL1+rPjSOGzra9wtFE/IzPnLPnsUTjJJh+1c+pR+UM/7X92G63yX+syuYOAqWKvjbVZ4CgWwDdd6Pn3bEcROkEI4wocgu8oKE4QtXqTS7DRzUqzFMUDyVcaYhO7WWCRBYxEjLBOTUY3EQWdeuy5453dRqtETHOtsLXfLHBKCxAR+Nspj1SDF9HH1Idx4pjlke8q6Nw2Witiugrjb0r+by3gHZT3SKcixzpHlHbI/M5qoy1iOuTVbX2v1gJNFOPn91U/giDfDy+ti+trMdo5bfQvZXcquw/qYjEAAAAASUVORK5CYII=\n", + "text/latex": [ + "$\\displaystyle 1 + 16 e^{- 2 \\sin^{2}{\\left(4 x \\right)}} \\sin{\\left(4 x \\right)} \\cos{\\left(4 x \\right)}$" + ], + "text/plain": [ + " 2 \n", + " -2⋅sin (4⋅x) \n", + "1 + 16⋅ℯ ⋅sin(4⋅x)⋅cos(4⋅x)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sym.diff(f, x)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/lecture11/sympy/jacobian.py b/lecture11/sympy/jacobian.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# File : jacobian.py +# Description: Compute the Jacobian of f(x) symbolically +# Copyright 2021 Harvard University. All Rights Reserved. +import sympy as sym + +x = sym.symbols('x') # define the symbolic variable +sym.init_printing(use_unicode=True) # for pretty terminal printing + +f = x - sym.exp(-2 * sym.sin(4 * x)**2) # our function f(x) +J = sym.diff(f, x) # compute the derivative w/r/t x +print(J) diff --git a/lecture14/CI_tests/.github/workflows/lecture14_intro.yml b/lecture14/CI_tests/.github/workflows/lecture14_intro.yml @@ -0,0 +1,44 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture14 Continuous Integration Introduction + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the main + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a job called "bash_command" + bash_command: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + - name: Bash commands + run: echo "Hello CI!" + + # another job called "bash_script" + bash_script: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to + # it) See: https://github.com/actions/checkout + - uses: actions/checkout@v3 # uses a GitHub action + - name: Bash script + run: ./script.sh # must be executable and in repository root diff --git a/lecture14/CI_tests/.github/workflows/lecture14_tests.yml b/lecture14/CI_tests/.github/workflows/lecture14_tests.yml @@ -0,0 +1,53 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture14 Continuous Integration Tests + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a single job called "install_and_test" + install_and_test: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the + # job. For more on steps: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to it) + # See: https://github.com/actions/checkout + - uses: actions/checkout@v3 + + # Enable Python environment in your CI container + # See: https://github.com/actions/setup-python + - uses: actions/setup-python@v3 + with: + python-version: '3.10' # let's use a recent version + + # Install Python dependencies + - name: Install dependencies + run: python -m pip install build pytest + + # Build and install our package in the container + - name: Build and install the cs107_project in the container (using PEP517/518) + run: (python -m build --wheel && python -m pip install dist/*) + + # Run the tests for the installed package + - name: Run tests using test harness + run: (cd tests && ./run_tests.sh CI) diff --git a/lecture14/CI_tests/.gitignore b/lecture14/CI_tests/.gitignore @@ -0,0 +1,6 @@ +/dist +*.pyc +*.egg-info +**/__pycache__ +**/.pytest* +.coverage diff --git a/lecture14/CI_tests/LICENSE b/lecture14/CI_tests/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Harvard University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lecture14/CI_tests/README.rst b/lecture14/CI_tests/README.rst @@ -0,0 +1,32 @@ +Test project for continuous integration and testing +=================================================== + +Test project continuation lecture 14: adding CI and Python ``unittest`` and +``pytest`` tests. + + +.. code:: console + + . + ├── LICENSE + ├── pyproject.toml + ├── README.md + ├── src + │  └── cs107_package + │  ├── __init__.py + │  ├── __main__.py + │  ├── subpkg_1 + │  │  ├── __init__.py + │  │  ├── module_1.py + │  │  └── module_2.py + │  └── subpkg_2 + │  ├── __init__.py + │  ├── module_3.py + │  ├── module_4.py + │  └── module_5.py + └── tests + ├── run_coverage.sh + ├── run_tests.sh + └── subpkg_1 + ├── test_module_1.py + └── test_module_2.py diff --git a/lecture14/CI_tests/pyproject.toml b/lecture14/CI_tests/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cs107_package" +version = "0.0.1" +authors = [ + { name="Fabian Wermelinger", email="fabianw@seas.harvard.edu" }, +] +description = "Test project for continuous integration and testing" +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Testing" +] + +[project.urls] +"Homepage" = "https://harvard-iacs.github.io/2022-CS107/" diff --git a/lecture14/CI_tests/script.sh b/lecture14/CI_tests/script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo "Hello CI!" +exit 0 diff --git a/lecture14/CI_tests/src/cs107_package/__init__.py b/lecture14/CI_tests/src/cs107_package/__init__.py @@ -0,0 +1,4 @@ +from .subpkg_1 import (foo, bar) +from .subpkg_2 import baz + +__all__ = ['foo', 'bar', 'baz'] diff --git a/lecture14/CI_tests/src/cs107_package/__main__.py b/lecture14/CI_tests/src/cs107_package/__main__.py @@ -0,0 +1,6 @@ +import datetime as dt +import numpy as np + +print( + f"Hello from cs107_package! Today is: {dt.datetime.now()} and pi = {np.pi:e}" +) diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_1/__init__.py b/lecture14/CI_tests/src/cs107_package/subpkg_1/__init__.py @@ -0,0 +1,4 @@ +from .module_1 import foo +from .module_2 import bar + +__all__ = ['foo', 'bar'] diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_1/module_1.py b/lecture14/CI_tests/src/cs107_package/subpkg_1/module_1.py @@ -0,0 +1,8 @@ +class Foo: + def __init__(self, a, b): + self.a = a + self.b = b + + +def foo(): + return "cs107_package.subpkg_1.module_1.foo()" diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_1/module_2.py b/lecture14/CI_tests/src/cs107_package/subpkg_1/module_2.py @@ -0,0 +1,6 @@ +from ..subpkg_2 import module_3 as mod3 + + +def bar(): + mod3.baz(0) + return "cs107_package.subpkg_1.module_2.bar()" diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_2/__init__.py b/lecture14/CI_tests/src/cs107_package/subpkg_2/__init__.py @@ -0,0 +1,3 @@ +from .module_3 import baz + +__all__ = ['baz'] diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_2/module_3.py b/lecture14/CI_tests/src/cs107_package/subpkg_2/module_3.py @@ -0,0 +1,37 @@ +""" +This is the docstring for ./subpkg_2/module_3.py. This module provides one +function `baz`. Example usage is: + +>>> baz(0) +0 +""" + + +def baz(x): + """ + Return the input x if it is an int or float. + + Parameters + ---------- + x : input argument + + Returns + ------- + x : If it is of type int or float + + Examples + -------- + >>> baz(0) + 0 + + >>> baz(0.0) + 0.0 + + >>> baz('a string') + Traceback (most recent call last): + ... + ValueError: x must be int or float + """ + if not isinstance(x, (int, float)): + raise ValueError('x must be int or float') + return x diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_2/module_4.py b/lecture14/CI_tests/src/cs107_package/subpkg_2/module_4.py @@ -0,0 +1 @@ +pass diff --git a/lecture14/CI_tests/src/cs107_package/subpkg_2/module_5.py b/lecture14/CI_tests/src/cs107_package/subpkg_2/module_5.py @@ -0,0 +1 @@ +pass diff --git a/lecture14/CI_tests/tests/run_tests.sh b/lecture14/CI_tests/tests/run_tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# File : run_tests.sh +# Description: Test suite driver script +# Copyright 2022 Harvard University. All Rights Reserved. +set -e + +# list of test cases you want to run +tests=( + # test_other_things_on_root_level.py + subpkg_1/test_module_1.py + subpkg_1/test_module_2.py + # subpkg_2/test_module_3.py + # subpkg_2/test_module_4.py +) + +# Must add the module source path because we use `import cs107_package` in +# our test suite. This is necessary if you want to test in your local +# development environment without properly installing the package. +export PYTHONPATH="$(pwd -P)/../src":${PYTHONPATH} + +# decide what driver to use (depending on arguments given) +if [[ $# -gt 0 && ${1} == 'coverage' ]]; then + driver="${@} -m unittest" +elif [[ $# -gt 0 && ${1} == 'pytest' ]]; then + driver="${@}" +elif [[ $# -gt 0 && ${1} == 'CI' ]]; then + # Assumes the package has been installed and dependencies resolved. This + # would be the situation for a customer. Uses `pytest` for testing. + shift + unset PYTHONPATH + driver="pytest ${@}" +else + driver="python ${@} -m unittest" +fi + +# run the tests +${driver} ${tests[@]} diff --git a/lecture14/CI_tests/tests/subpkg_1/test_module_1.py b/lecture14/CI_tests/tests/subpkg_1/test_module_1.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_1 of the +cs107_package. +""" + +# Here we use `unittest` from the Python standard library +import unittest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_1 import (Foo, foo) + + +# test classes must start with `Test` (classes are required for `unittest` tests) +class TestTypes( + unittest.TestCase +): # test classes must inherit from unittest.TestCase + + # test methods (functions) must be prepended with `test_`. + def test_class_Foo(self): + """ + This is just a trivial test to check that `Foo` is initialized + correctly. More tests associated to the class `Foo` could be written in + this method. + """ + f = Foo(1, 2) # create instance + self.assertEqual(f.a, 1) # check attribute `a` + self.assertEqual(f.b, 2) # check attribute `b` + + +class TestFunctions(unittest.TestCase): + + def test_function_foo(self): + """ + This is just a trivial test to check the return value of function `foo`. + """ + # assert the return value of foo() + self.assertEqual(foo(), "cs107_package.subpkg_1.module_1.foo()") + + +if __name__ == '__main__': + # can use this to run the test module standalone + unittest.main() diff --git a/lecture14/CI_tests/tests/subpkg_1/test_module_2.py b/lecture14/CI_tests/tests/subpkg_1/test_module_2.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_2 of the +cs107_package. +""" + +# Let us use `pytest` this time +import pytest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_2 import (bar) + + +class TestFunctions: + """We do not inherit from unittest.TestCase for pytest's!""" + + def test_bar(self): + """ + This is just a trivial test to check the return value of function `bar`. + """ + # assert the return value of bar() (note that this uses Python's + # `assert` statement directly, no need to inherit from anything!) + assert bar() == "cs107_package.subpkg_1.module_2.bar()" + + +# ============================================================================== +# A test function unrelated to `cs107_package`. It is here to demonstrate the +# feature of `pytest` used in `test_example_function` below. +def example_function(): + """If you have code that raises exceptions, pytest can verify them.""" + raise RuntimeError("This function should not be called") + + +def test_example_function(): + with pytest.raises(RuntimeError): + example_function() + + +# ============================================================================== + +if __name__ == "__main__": + # can use this to run the test module standalone + pytest.main() diff --git a/lecture15/CI_tests/.github/workflows/lecture14_intro.yml b/lecture15/CI_tests/.github/workflows/lecture14_intro.yml @@ -0,0 +1,44 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture14 Continuous Integration Introduction + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the main + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a job called "bash_command" + bash_command: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + - name: Bash commands + run: echo "Hello CI!" + + # another job called "bash_script" + bash_script: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to + # it) See: https://github.com/actions/checkout + - uses: actions/checkout@v3 # uses a GitHub action + - name: Bash script + run: ./script.sh # must be executable and in repository root diff --git a/lecture15/CI_tests/.github/workflows/lecture14_tests.yml b/lecture15/CI_tests/.github/workflows/lecture14_tests.yml @@ -0,0 +1,53 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture14 Continuous Integration Tests + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a single job called "install_and_test" + install_and_test: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the + # job. For more on steps: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to it) + # See: https://github.com/actions/checkout + - uses: actions/checkout@v3 + + # Enable Python environment in your CI container + # See: https://github.com/actions/setup-python + - uses: actions/setup-python@v3 + with: + python-version: '3.10' # let's use a recent version + + # Install Python dependencies + - name: Install dependencies + run: python -m pip install build pytest + + # Build and install our package in the container + - name: Build and install the cs107_project in the container (using PEP517/518) + run: (python -m build --wheel && python -m pip install dist/*) + + # Run the tests for the installed package + - name: Run tests using test harness + run: (cd tests && ./run_tests.sh CI) diff --git a/lecture15/CI_tests/.github/workflows/lecture15_coverage.yml b/lecture15/CI_tests/.github/workflows/lecture15_coverage.yml @@ -0,0 +1,64 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture15 Continuous Integration Test Coverage + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a single job called "test_coverage" + test_coverage: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the + # job. For more on steps: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to it) + # See: https://github.com/actions/checkout + - uses: actions/checkout@v3 + + # Enable Python environment in your CI container + # See: https://github.com/actions/setup-python + - uses: actions/setup-python@v3 + with: + python-version: '3.10' # let's use a recent version + + # Install Python dependencies + - name: Install dependencies + run: python -m pip install build pytest pytest-cov + + # Build and install our package in the container + - name: Build and install the cs107_project in the container (using PEP517/518) + run: (python -m build --wheel && python -m pip install dist/*) + + # Run the test coverage for the build + - name: Run tests and generate coverage html + run: (cd tests && ./run_tests.sh CI --cov=cs107_package --cov-report=html:htmlcov) + + # Remove .gitignore file in test coverage data to be pushed to gh-pages + # branch + - name: Clean .gitignore in coverage output + run: rm -f tests/htmlcov/.gitignore + + # Deploy to gh-pages branch + - name: Deploy test coverage GitHub page + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: tests/htmlcov diff --git a/lecture15/CI_tests/.gitignore b/lecture15/CI_tests/.gitignore @@ -0,0 +1,6 @@ +/dist +*.pyc +*.egg-info +**/__pycache__ +**/.pytest* +.coverage diff --git a/lecture15/CI_tests/LICENSE b/lecture15/CI_tests/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Harvard University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lecture15/CI_tests/README.rst b/lecture15/CI_tests/README.rst @@ -0,0 +1,38 @@ +Test project for continuous integration and testing +=================================================== + +Test project continuation lecture 14/15: adding CI and Python ``unittest`` and +``pytest`` tests. + + +.. code:: console + + . + ├── docs + │  ├── conf.py + │  ├── index.rst + │  ├── make.bat + │  ├── Makefile + │  └── source + ├── LICENSE + ├── pyproject.toml + ├── README.md + ├── src + │  └── cs107_package + │  ├── __init__.py + │  ├── __main__.py + │  ├── subpkg_1 + │  │  ├── __init__.py + │  │  ├── module_1.py + │  │  └── module_2.py + │  └── subpkg_2 + │  ├── __init__.py + │  ├── module_3.py + │  ├── module_4.py + │  └── module_5.py + └── tests + ├── run_coverage.sh + ├── run_tests.sh + └── subpkg_1 + ├── test_module_1.py + └── test_module_2.py diff --git a/lecture15/CI_tests/docs/Makefile b/lecture15/CI_tests/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/lecture15/CI_tests/docs/conf.py b/lecture15/CI_tests/docs/conf.py @@ -0,0 +1,62 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath('../src')) + + +# -- Project information ----------------------------------------------------- + +project = 'CS107 Sample Project' +copyright = '2022, CS107/AC207 Teaching Staff' +author = 'Fabian Wermelinger' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon' +] + +add_module_names = False + +autosummary_generate = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +import sphinx_rtd_theme + +pygments_style = 'sphinx' +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/lecture15/CI_tests/docs/index.rst b/lecture15/CI_tests/docs/index.rst @@ -0,0 +1,23 @@ +.. CS107 Sample Project documentation master file, created by + sphinx-quickstart on Sat Oct 9 16:22:25 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to CS107 Sample Project's documentation! +================================================ + +.. include:: ../README.rst + +.. toctree:: + :maxdepth: 1 + :caption: Docstring + + source/docstring/cs107_package + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/lecture15/CI_tests/docs/make.bat b/lecture15/CI_tests/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/lecture15/CI_tests/docs/source/docstring/cs107_package.rst b/lecture15/CI_tests/docs/source/docstring/cs107_package.rst @@ -0,0 +1,10 @@ +cs107\_package package +====================== + +Module contents +--------------- + +.. automodule:: cs107_package + :members: + :undoc-members: + :show-inheritance: diff --git a/lecture15/CI_tests/pyproject.toml b/lecture15/CI_tests/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cs107_package" +version = "0.0.1" +authors = [ + { name="Fabian Wermelinger", email="fabianw@seas.harvard.edu" }, +] +description = "Test project for continuous integration and testing" +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Testing" +] + +[project.urls] +"Homepage" = "https://harvard-iacs.github.io/2022-CS107/" diff --git a/lecture15/CI_tests/script.sh b/lecture15/CI_tests/script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo "Hello CI!" +exit 0 diff --git a/lecture15/CI_tests/src/cs107_package/__init__.py b/lecture15/CI_tests/src/cs107_package/__init__.py @@ -0,0 +1,12 @@ +"""Docstring for cs107_package test package + +Some Python features were discussed using this package related to packaging and +distribution of Python packages in lecture 9. + +""" + +from .subpkg_1 import (foo, bar) +from .subpkg_2 import baz +from .example import foo as numpy_example + +__all__ = ['foo', 'bar', 'baz', 'numpy_example'] diff --git a/lecture15/CI_tests/src/cs107_package/__main__.py b/lecture15/CI_tests/src/cs107_package/__main__.py @@ -0,0 +1,6 @@ +import datetime as dt +import numpy as np + +print( + f"Hello from cs107_package! Today is: {dt.datetime.now()} and pi = {np.pi:e}" +) diff --git a/lecture15/CI_tests/src/cs107_package/example.py b/lecture15/CI_tests/src/cs107_package/example.py @@ -0,0 +1,136 @@ +"""Docstring for the example.py module. + +Modules names should have short, all-lowercase names. The module name may +have underscores if this improves readability. + +Every module should have a docstring at the very top of the file. The +module's docstring may extend over multiple lines. If your docstring does +extend over multiple lines, the closing three quotation marks must be on +a line by itself, preferably preceded by a blank line. + +This file is from https://github.com/numpy/numpydoc/blob/main/doc/example.py + +Rendered: https://numpydoc.readthedocs.io/en/v1.5.0/example.html#module-example + +""" + +import os # standard library imports first + +# Do NOT import using *, e.g. from numpy import * +# +# Import the module using +# +# import numpy +# +# instead or import individual functions as needed, e.g +# +# from numpy import array, zeros +# +# If you prefer the use of abbreviated module names, we suggest the +# convention used by NumPy itself:: + +import numpy as np +import matplotlib as mpl +import matplotlib.pyplot as plt + +# These abbreviated names are not to be used in docstrings; users must +# be able to paste and execute docstrings after importing only the +# numpy module itself, unabbreviated. + + +def foo(var1, var2, *args, long_var_name="hi", only_seldom_used_keyword=0, **kwargs): + r"""Summarize the function in one line. + + Several sentences providing an extended description. Refer to + variables using back-ticks, e.g. `var`. + + Parameters + ---------- + var1 : array_like + Array_like means all those objects -- lists, nested lists, etc. -- + that can be converted to an array. We can also refer to + variables like `var1`. + var2 : int + The type above can either refer to an actual Python type + (e.g. ``int``), or describe the type of the variable in more + detail, e.g. ``(N,) ndarray`` or ``array_like``. + *args : iterable + Other arguments. + long_var_name : {'hi', 'ho'}, optional + Choices in brackets, default first when optional. + + Returns + ------- + type + Explanation of anonymous return value of type ``type``. + describe : type + Explanation of return value named `describe`. + out : type + Explanation of `out`. + type_without_description + + Other Parameters + ---------------- + only_seldom_used_keyword : int, optional + Infrequently used parameters can be described under this optional + section to prevent cluttering the Parameters section. + **kwargs : dict + Other infrequently used keyword arguments. Note that all keyword + arguments appearing after the first parameter specified under the + Other Parameters section, should also be described under this + section. + + Raises + ------ + BadException + Because you shouldn't have done that. + + See Also + -------- + numpy.array : Relationship (optional). + numpy.ndarray : Relationship (optional), which could be fairly long, in + which case the line wraps here. + numpy.dot, numpy.linalg.norm, numpy.eye + + Notes + ----- + Notes about the implementation algorithm (if needed). + + This can have multiple paragraphs. + + You may include some math: + + .. math:: X(e^{j\omega } ) = x(n)e^{ - j\omega n} + + And even use a Greek symbol like :math:`\omega` inline. + + References + ---------- + Cite the relevant literature, e.g. [1]_. You may also cite these + references in the notes section above. + + .. [1] O. McNoleg, "The integration of GIS, remote sensing, + expert systems and adaptive co-kriging for environmental habitat + modelling of the Highland Haggis using object-oriented, fuzzy-logic + and neural-network techniques," Computers & Geosciences, vol. 22, + pp. 585-588, 1996. + + Examples + -------- + These are written in doctest format, and should illustrate how to + use the function. + + >>> a = [1, 2, 3] + >>> print([x + 3 for x in a]) + [4, 5, 6] + >>> print("a\nb") + a + b + >>> foo(1, 2) # XXX: [fabianw@seas.harvard.edu; 2022-10-19] call this function + 3 + """ + # After closing class docstring, there should be one blank line to + # separate following codes (according to PEP257). + # But for function, method and module, there should be no blank lines + # after closing the docstring. + return var1 + var2 diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_1/__init__.py b/lecture15/CI_tests/src/cs107_package/subpkg_1/__init__.py @@ -0,0 +1,4 @@ +from .module_1 import foo +from .module_2 import bar + +__all__ = ['foo', 'bar'] diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_1/module_1.py b/lecture15/CI_tests/src/cs107_package/subpkg_1/module_1.py @@ -0,0 +1,8 @@ +class Foo: + def __init__(self, a, b): + self.a = a + self.b = b + + +def foo(): + return "cs107_package.subpkg_1.module_1.foo()" diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_1/module_2.py b/lecture15/CI_tests/src/cs107_package/subpkg_1/module_2.py @@ -0,0 +1,6 @@ +from ..subpkg_2 import module_3 as mod3 + + +def bar(): + mod3.baz(0) + return "cs107_package.subpkg_1.module_2.bar()" diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_2/__init__.py b/lecture15/CI_tests/src/cs107_package/subpkg_2/__init__.py @@ -0,0 +1,3 @@ +from .module_3 import baz + +__all__ = ['baz'] diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_2/module_3.py b/lecture15/CI_tests/src/cs107_package/subpkg_2/module_3.py @@ -0,0 +1,37 @@ +""" +This is the docstring for ./subpkg_2/module_3.py. This module provides one +function `baz`. Example usage is: + +>>> baz(0) +0 +""" + + +def baz(x): + """ + Return the input x if it is an int or float. + + Parameters + ---------- + x : input argument + + Returns + ------- + x : If it is of type int or float + + Examples + -------- + >>> baz(0) + 0 + + >>> baz(0.0) + 0.0 + + >>> baz('a string') + Traceback (most recent call last): + ... + ValueError: x must be int or float + """ + if not isinstance(x, (int, float)): + raise ValueError('x must be int or float') + return x diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_2/module_4.py b/lecture15/CI_tests/src/cs107_package/subpkg_2/module_4.py @@ -0,0 +1 @@ +pass diff --git a/lecture15/CI_tests/src/cs107_package/subpkg_2/module_5.py b/lecture15/CI_tests/src/cs107_package/subpkg_2/module_5.py @@ -0,0 +1 @@ +pass diff --git a/lecture15/CI_tests/tests/check_coverage.sh b/lecture15/CI_tests/tests/check_coverage.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# File : check_coverage.sh +# Description: Coverage wrapper around test suite driver script +# Copyright 2022 Harvard University. All Rights Reserved. +set -e + +tool='coverage' +if [[ $# -gt 0 ]]; then + # optional argument to use different tool to check coverage + tool="${1}"; shift +fi + +if [[ ${tool} == 'coverage' ]]; then + # run the tests (generates coverage data to build report) + ./run_tests.sh coverage run --source=cs107_package "${@}" + + # build the coverage report on stdout + coverage report -m +elif [[ ${tool} == 'pytest' ]]; then + # generate coverage reports with pytest in one go + ./run_tests.sh pytest --cov=cs107_package "${@}" +else + # error: write to stderr + >&2 echo "Error: unknown tool '${tool}'" + exit 1 +fi diff --git a/lecture15/CI_tests/tests/run_tests.sh b/lecture15/CI_tests/tests/run_tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# File : run_tests.sh +# Description: Test suite driver script +# Copyright 2022 Harvard University. All Rights Reserved. +set -e + +# list of test cases you want to run +tests=( + # test_other_things_on_root_level.py + subpkg_1/test_module_1.py + subpkg_1/test_module_2.py + # subpkg_2/test_module_3.py + # subpkg_2/test_module_4.py +) + +# Must add the module source path because we use `import cs107_package` in +# our test suite. This is necessary if you want to test in your local +# development environment without properly installing the package. +export PYTHONPATH="$(pwd -P)/../src":${PYTHONPATH} + +# decide what driver to use (depending on arguments given) +if [[ $# -gt 0 && ${1} == 'coverage' ]]; then + driver="${@} -m unittest" +elif [[ $# -gt 0 && ${1} == 'pytest' ]]; then + driver="${@}" +elif [[ $# -gt 0 && ${1} == 'CI' ]]; then + # Assumes the package has been installed and dependencies resolved. This + # would be the situation for a customer. Uses `pytest` for testing. + shift + unset PYTHONPATH + driver="pytest ${@}" +else + driver="python ${@} -m unittest" +fi + +# run the tests +${driver} ${tests[@]} diff --git a/lecture15/CI_tests/tests/subpkg_1/test_module_1.py b/lecture15/CI_tests/tests/subpkg_1/test_module_1.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_1 of the +cs107_package. +""" + +# Here we use `unittest` from the Python standard library +import unittest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_1 import (Foo, foo) + + +# test classes must start with `Test` (classes are required for `unittest` tests) +class TestTypes( + unittest.TestCase +): # test classes must inherit from unittest.TestCase + + # test methods (functions) must be prepended with `test_`. + def test_class_Foo(self): + """ + This is just a trivial test to check that `Foo` is initialized + correctly. More tests associated to the class `Foo` could be written in + this method. + """ + f = Foo(1, 2) # create instance + self.assertEqual(f.a, 1) # check attribute `a` + self.assertEqual(f.b, 2) # check attribute `b` + + +class TestFunctions(unittest.TestCase): + + def test_function_foo(self): + """ + This is just a trivial test to check the return value of function `foo`. + """ + # assert the return value of foo() + self.assertEqual(foo(), "cs107_package.subpkg_1.module_1.foo()") + + +if __name__ == '__main__': + # can use this to run the test module standalone + unittest.main() diff --git a/lecture15/CI_tests/tests/subpkg_1/test_module_2.py b/lecture15/CI_tests/tests/subpkg_1/test_module_2.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_2 of the +cs107_package. +""" + +# Let us use `pytest` this time +import pytest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_2 import (bar) + + +class TestFunctions: + """We do not inherit from unittest.TestCase for pytest's!""" + + def test_bar(self): + """ + This is just a trivial test to check the return value of function `bar`. + """ + # assert the return value of bar() (note that this uses Python's + # `assert` statement directly, no need to inherit from anything!) + assert bar() == "cs107_package.subpkg_1.module_2.bar()" + + +# ============================================================================== +# A test function unrelated to `cs107_package`. It is here to demonstrate the +# feature of `pytest` used in `test_example_function` below. +def example_function(): + """If you have code that raises exceptions, pytest can verify them.""" + raise RuntimeError("This function should not be called") + + +def test_example_function(): + with pytest.raises(RuntimeError): + example_function() + + +# ============================================================================== + +if __name__ == "__main__": + # can use this to run the test module standalone + pytest.main() diff --git a/lecture16/CI_tests/.github/workflows/lecture16.yml b/lecture16/CI_tests/.github/workflows/lecture16.yml @@ -0,0 +1,56 @@ +# This is a basic workflow to help you get started with Actions +# You find out more at: https://docs.github.com/en/actions +name: Lecture16 Continuous Integration Test Coverage (with custom container) + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master + # branch + push: + branches: + - main + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel. For more on jobs: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobs +jobs: + # This workflow contains a single job called "test_coverage" + test_coverage: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # here we specify the container image that we have built ourselves. The + # image already contains the Python environment for our needs and we can + # skip to install a Python environment and dependencies below. Procedure is + # similar for other CI providers. + # https://hub.docker.com/r/iacs/cs107_lecture16/tags + container: + image: iacs/cs107_lecture16:latest + + # Steps represent a sequence of tasks that will be executed as part of the + # job. For more on steps: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsteps + steps: + # Check out your repository under $GITHUB_WORKSPACE (job needs access to it) + # See: https://github.com/actions/checkout + - uses: actions/checkout@v3 + + # Check what Python installation we are using (installed in custom + # container) + - name: Check Python installation + run: python3 -VV + + # Build and install our package in the container + - name: Build and install the cs107_project in the container (using PEP517/518) + run: (python3 -m build --wheel && python3 -m pip install dist/*) + + # Run the test coverage for the build + - name: Run tests and generate coverage html + run: (cd tests && ./run_tests.sh CI --cov=cs107_package --cov-report=term-missing) diff --git a/lecture16/CI_tests/.gitignore b/lecture16/CI_tests/.gitignore @@ -0,0 +1,6 @@ +/dist +*.pyc +*.egg-info +**/__pycache__ +**/.pytest* +.coverage diff --git a/lecture16/CI_tests/LICENSE b/lecture16/CI_tests/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Harvard University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lecture16/CI_tests/README.rst b/lecture16/CI_tests/README.rst @@ -0,0 +1,38 @@ +Test project for continuous integration and testing +=================================================== + +Test project continuation lecture 14/15: adding CI and Python ``unittest`` and +``pytest`` tests. + + +.. code:: console + + . + ├── docs + │  ├── conf.py + │  ├── index.rst + │  ├── make.bat + │  ├── Makefile + │  └── source + ├── LICENSE + ├── pyproject.toml + ├── README.md + ├── src + │  └── cs107_package + │  ├── __init__.py + │  ├── __main__.py + │  ├── subpkg_1 + │  │  ├── __init__.py + │  │  ├── module_1.py + │  │  └── module_2.py + │  └── subpkg_2 + │  ├── __init__.py + │  ├── module_3.py + │  ├── module_4.py + │  └── module_5.py + └── tests + ├── run_coverage.sh + ├── run_tests.sh + └── subpkg_1 + ├── test_module_1.py + └── test_module_2.py diff --git a/lecture16/CI_tests/docs/Makefile b/lecture16/CI_tests/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/lecture16/CI_tests/docs/conf.py b/lecture16/CI_tests/docs/conf.py @@ -0,0 +1,62 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath('../src')) + + +# -- Project information ----------------------------------------------------- + +project = 'CS107 Sample Project' +copyright = '2022, CS107/AC207 Teaching Staff' +author = 'Fabian Wermelinger' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon' +] + +add_module_names = False + +autosummary_generate = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +import sphinx_rtd_theme + +pygments_style = 'sphinx' +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/lecture16/CI_tests/docs/index.rst b/lecture16/CI_tests/docs/index.rst @@ -0,0 +1,23 @@ +.. CS107 Sample Project documentation master file, created by + sphinx-quickstart on Sat Oct 9 16:22:25 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to CS107 Sample Project's documentation! +================================================ + +.. include:: ../README.rst + +.. toctree:: + :maxdepth: 1 + :caption: Docstring + + source/docstring/cs107_package + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/lecture16/CI_tests/docs/make.bat b/lecture16/CI_tests/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/lecture16/CI_tests/docs/source/docstring/cs107_package.rst b/lecture16/CI_tests/docs/source/docstring/cs107_package.rst @@ -0,0 +1,10 @@ +cs107\_package package +====================== + +Module contents +--------------- + +.. automodule:: cs107_package + :members: + :undoc-members: + :show-inheritance: diff --git a/lecture16/CI_tests/pyproject.toml b/lecture16/CI_tests/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cs107_package" +version = "0.0.1" +authors = [ + { name="Fabian Wermelinger", email="fabianw@seas.harvard.edu" }, +] +description = "Test project for continuous integration and testing" +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Testing" +] + +[project.urls] +"Homepage" = "https://harvard-iacs.github.io/2022-CS107/" diff --git a/lecture16/CI_tests/script.sh b/lecture16/CI_tests/script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +echo "Hello CI!" +exit 0 diff --git a/lecture16/CI_tests/src/cs107_package/__init__.py b/lecture16/CI_tests/src/cs107_package/__init__.py @@ -0,0 +1,12 @@ +"""Docstring for cs107_package test package + +Some Python features were discussed using this package related to packaging and +distribution of Python packages in lecture 9. + +""" + +from .subpkg_1 import (foo, bar) +from .subpkg_2 import baz +from .example import foo as numpy_example + +__all__ = ['foo', 'bar', 'baz', 'numpy_example'] diff --git a/lecture16/CI_tests/src/cs107_package/__main__.py b/lecture16/CI_tests/src/cs107_package/__main__.py @@ -0,0 +1,6 @@ +import datetime as dt +import numpy as np + +print( + f"Hello from cs107_package! Today is: {dt.datetime.now()} and pi = {np.pi:e}" +) diff --git a/lecture16/CI_tests/src/cs107_package/example.py b/lecture16/CI_tests/src/cs107_package/example.py @@ -0,0 +1,136 @@ +"""Docstring for the example.py module. + +Modules names should have short, all-lowercase names. The module name may +have underscores if this improves readability. + +Every module should have a docstring at the very top of the file. The +module's docstring may extend over multiple lines. If your docstring does +extend over multiple lines, the closing three quotation marks must be on +a line by itself, preferably preceded by a blank line. + +This file is from https://github.com/numpy/numpydoc/blob/main/doc/example.py + +Rendered: https://numpydoc.readthedocs.io/en/v1.5.0/example.html#module-example + +""" + +import os # standard library imports first + +# Do NOT import using *, e.g. from numpy import * +# +# Import the module using +# +# import numpy +# +# instead or import individual functions as needed, e.g +# +# from numpy import array, zeros +# +# If you prefer the use of abbreviated module names, we suggest the +# convention used by NumPy itself:: + +import numpy as np +import matplotlib as mpl +import matplotlib.pyplot as plt + +# These abbreviated names are not to be used in docstrings; users must +# be able to paste and execute docstrings after importing only the +# numpy module itself, unabbreviated. + + +def foo(var1, var2, *args, long_var_name="hi", only_seldom_used_keyword=0, **kwargs): + r"""Summarize the function in one line. + + Several sentences providing an extended description. Refer to + variables using back-ticks, e.g. `var`. + + Parameters + ---------- + var1 : array_like + Array_like means all those objects -- lists, nested lists, etc. -- + that can be converted to an array. We can also refer to + variables like `var1`. + var2 : int + The type above can either refer to an actual Python type + (e.g. ``int``), or describe the type of the variable in more + detail, e.g. ``(N,) ndarray`` or ``array_like``. + *args : iterable + Other arguments. + long_var_name : {'hi', 'ho'}, optional + Choices in brackets, default first when optional. + + Returns + ------- + type + Explanation of anonymous return value of type ``type``. + describe : type + Explanation of return value named `describe`. + out : type + Explanation of `out`. + type_without_description + + Other Parameters + ---------------- + only_seldom_used_keyword : int, optional + Infrequently used parameters can be described under this optional + section to prevent cluttering the Parameters section. + **kwargs : dict + Other infrequently used keyword arguments. Note that all keyword + arguments appearing after the first parameter specified under the + Other Parameters section, should also be described under this + section. + + Raises + ------ + BadException + Because you shouldn't have done that. + + See Also + -------- + numpy.array : Relationship (optional). + numpy.ndarray : Relationship (optional), which could be fairly long, in + which case the line wraps here. + numpy.dot, numpy.linalg.norm, numpy.eye + + Notes + ----- + Notes about the implementation algorithm (if needed). + + This can have multiple paragraphs. + + You may include some math: + + .. math:: X(e^{j\omega } ) = x(n)e^{ - j\omega n} + + And even use a Greek symbol like :math:`\omega` inline. + + References + ---------- + Cite the relevant literature, e.g. [1]_. You may also cite these + references in the notes section above. + + .. [1] O. McNoleg, "The integration of GIS, remote sensing, + expert systems and adaptive co-kriging for environmental habitat + modelling of the Highland Haggis using object-oriented, fuzzy-logic + and neural-network techniques," Computers & Geosciences, vol. 22, + pp. 585-588, 1996. + + Examples + -------- + These are written in doctest format, and should illustrate how to + use the function. + + >>> a = [1, 2, 3] + >>> print([x + 3 for x in a]) + [4, 5, 6] + >>> print("a\nb") + a + b + >>> foo(1, 2) # XXX: [fabianw@seas.harvard.edu; 2022-10-19] call this function + 3 + """ + # After closing class docstring, there should be one blank line to + # separate following codes (according to PEP257). + # But for function, method and module, there should be no blank lines + # after closing the docstring. + return var1 + var2 diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_1/__init__.py b/lecture16/CI_tests/src/cs107_package/subpkg_1/__init__.py @@ -0,0 +1,4 @@ +from .module_1 import foo +from .module_2 import bar + +__all__ = ['foo', 'bar'] diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_1/module_1.py b/lecture16/CI_tests/src/cs107_package/subpkg_1/module_1.py @@ -0,0 +1,8 @@ +class Foo: + def __init__(self, a, b): + self.a = a + self.b = b + + +def foo(): + return "cs107_package.subpkg_1.module_1.foo()" diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_1/module_2.py b/lecture16/CI_tests/src/cs107_package/subpkg_1/module_2.py @@ -0,0 +1,6 @@ +from ..subpkg_2 import module_3 as mod3 + + +def bar(): + mod3.baz(0) + return "cs107_package.subpkg_1.module_2.bar()" diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_2/__init__.py b/lecture16/CI_tests/src/cs107_package/subpkg_2/__init__.py @@ -0,0 +1,3 @@ +from .module_3 import baz + +__all__ = ['baz'] diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_2/module_3.py b/lecture16/CI_tests/src/cs107_package/subpkg_2/module_3.py @@ -0,0 +1,37 @@ +""" +This is the docstring for ./subpkg_2/module_3.py. This module provides one +function `baz`. Example usage is: + +>>> baz(0) +0 +""" + + +def baz(x): + """ + Return the input x if it is an int or float. + + Parameters + ---------- + x : input argument + + Returns + ------- + x : If it is of type int or float + + Examples + -------- + >>> baz(0) + 0 + + >>> baz(0.0) + 0.0 + + >>> baz('a string') + Traceback (most recent call last): + ... + ValueError: x must be int or float + """ + if not isinstance(x, (int, float)): + raise ValueError('x must be int or float') + return x diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_2/module_4.py b/lecture16/CI_tests/src/cs107_package/subpkg_2/module_4.py @@ -0,0 +1 @@ +pass diff --git a/lecture16/CI_tests/src/cs107_package/subpkg_2/module_5.py b/lecture16/CI_tests/src/cs107_package/subpkg_2/module_5.py @@ -0,0 +1 @@ +pass diff --git a/lecture16/CI_tests/tests/check_coverage.sh b/lecture16/CI_tests/tests/check_coverage.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# File : check_coverage.sh +# Description: Coverage wrapper around test suite driver script +# Copyright 2022 Harvard University. All Rights Reserved. +set -e + +tool='coverage' +if [[ $# -gt 0 ]]; then + # optional argument to use different tool to check coverage + tool="${1}"; shift +fi + +if [[ ${tool} == 'coverage' ]]; then + # run the tests (generates coverage data to build report) + ./run_tests.sh coverage run --source=cs107_package "${@}" + + # build the coverage report on stdout + coverage report -m +elif [[ ${tool} == 'pytest' ]]; then + # generate coverage reports with pytest in one go + ./run_tests.sh pytest --cov=cs107_package "${@}" +else + # error: write to stderr + >&2 echo "Error: unknown tool '${tool}'" + exit 1 +fi diff --git a/lecture16/CI_tests/tests/run_tests.sh b/lecture16/CI_tests/tests/run_tests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# File : run_tests.sh +# Description: Test suite driver script +# Copyright 2022 Harvard University. All Rights Reserved. +set -e + +# list of test cases you want to run +tests=( + # test_other_things_on_root_level.py + subpkg_1/test_module_1.py + subpkg_1/test_module_2.py + # subpkg_2/test_module_3.py + # subpkg_2/test_module_4.py +) + +# Must add the module source path because we use `import cs107_package` in +# our test suite. This is necessary if you want to test in your local +# development environment without properly installing the package. +export PYTHONPATH="$(pwd -P)/../src":${PYTHONPATH} + +# decide what driver to use (depending on arguments given) +if [[ $# -gt 0 && ${1} == 'coverage' ]]; then + driver="${@} -m unittest" +elif [[ $# -gt 0 && ${1} == 'pytest' ]]; then + driver="${@}" +elif [[ $# -gt 0 && ${1} == 'CI' ]]; then + # Assumes the package has been installed and dependencies resolved. This + # would be the situation for a customer. Uses `pytest` for testing. + shift + unset PYTHONPATH + driver="pytest ${@}" +else + driver="python3 ${@} -m unittest" +fi + +# run the tests +${driver} ${tests[@]} diff --git a/lecture16/CI_tests/tests/subpkg_1/test_module_1.py b/lecture16/CI_tests/tests/subpkg_1/test_module_1.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_1 of the +cs107_package. +""" + +# Here we use `unittest` from the Python standard library +import unittest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_1 import (Foo, foo) + + +# test classes must start with `Test` (classes are required for `unittest` tests) +class TestTypes( + unittest.TestCase +): # test classes must inherit from unittest.TestCase + + # test methods (functions) must be prepended with `test_`. + def test_class_Foo(self): + """ + This is just a trivial test to check that `Foo` is initialized + correctly. More tests associated to the class `Foo` could be written in + this method. + """ + f = Foo(1, 2) # create instance + self.assertEqual(f.a, 1) # check attribute `a` + self.assertEqual(f.b, 2) # check attribute `b` + + +class TestFunctions(unittest.TestCase): + + def test_function_foo(self): + """ + This is just a trivial test to check the return value of function `foo`. + """ + # assert the return value of foo() + self.assertEqual(foo(), "cs107_package.subpkg_1.module_1.foo()") + + +if __name__ == '__main__': + # can use this to run the test module standalone + unittest.main() diff --git a/lecture16/CI_tests/tests/subpkg_1/test_module_2.py b/lecture16/CI_tests/tests/subpkg_1/test_module_2.py @@ -0,0 +1,42 @@ +""" +This test suite (a module) runs tests for subpkg_1.module_2 of the +cs107_package. +""" + +# Let us use `pytest` this time +import pytest + +# project code to test (requires `cs107_package` to be in PYTHONPATH) +from cs107_package.subpkg_1.module_2 import (bar) + + +class TestFunctions: + """We do not inherit from unittest.TestCase for pytest's!""" + + def test_bar(self): + """ + This is just a trivial test to check the return value of function `bar`. + """ + # assert the return value of bar() (note that this uses Python's + # `assert` statement directly, no need to inherit from anything!) + assert bar() == "cs107_package.subpkg_1.module_2.bar()" + + +# ============================================================================== +# A test function unrelated to `cs107_package`. It is here to demonstrate the +# feature of `pytest` used in `test_example_function` below. +def example_function(): + """If you have code that raises exceptions, pytest can verify them.""" + raise RuntimeError("This function should not be called") + + +def test_example_function(): + with pytest.raises(RuntimeError): + example_function() + + +# ============================================================================== + +if __name__ == "__main__": + # can use this to run the test module standalone + pytest.main() diff --git a/lecture16/docker/Dockerfile b/lecture16/docker/Dockerfile @@ -0,0 +1,35 @@ +# base image +FROM docker.io/fedora:latest + +RUN dnf -y update + +# install basic Python development and frequently used packages +RUN dnf -y install \ + python-devel \ + python-build \ + python-numpy \ + python-pandas \ + python-matplotlib \ + python-pytest \ + python-pytest-cov + +# we could add other custom Python code if we needed to. For example the +# cs107_package from lecture 9 (we will not do it because we will build this +# package using this container!) +# +# RUN python3 -m pip install -i https://test.pypi.org/simple/ Fall2022-CS107 + +# you can COPY a local file into the container image (the file will be +# added to the image file system) +COPY local_file . + +# you can use any shell commands with the RUN command +RUN echo "Hello CS107/AC207 Docker!" >hello.txt + +# entry point is bash shell +CMD ["/bin/bash"] + +# add non-root user (optional, for GitHub Actions user must be root) +# RUN useradd -m cs107 +# WORKDIR /home/cs107 +# USER cs107 diff --git a/lecture16/docker/README.md b/lecture16/docker/README.md @@ -0,0 +1,7 @@ +The user for DockerHub in the shell scripts is `iacs` and you would have to +change this for your own deployments. + +[IACS DockerHub](https://hub.docker.com/u/iacs) + +See also this [Dockerfile +reference](https://docs.docker.com/engine/reference/builder/) diff --git a/lecture16/docker/build.sh b/lecture16/docker/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +export BUILDAH_FORMAT=docker +docker build -t 'iacs/cs107_lecture16:latest' . diff --git a/lecture16/docker/deploy.sh b/lecture16/docker/deploy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +# Always update this if you made changes to the Dockerfile +TAG=1.0.2 + +export BUILDAH_FORMAT=docker +docker build \ + -t "iacs/cs107_lecture16:${TAG}" \ + -t "iacs/cs107_lecture16:latest" . + +docker login docker.io +docker push "iacs/cs107_lecture16:${TAG}" +docker push "iacs/cs107_lecture16:latest" + +docker rmi -f iacs/cs107_lecture16:${TAG} diff --git a/lecture16/docker/local_file b/lecture16/docker/local_file @@ -0,0 +1 @@ +File contents you want to include in your Docker container diff --git a/lecture16/docker/remove.sh b/lecture16/docker/remove.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -e +docker rmi -f iacs/cs107_lecture16:latest diff --git a/lecture16/docker/run.sh b/lecture16/docker/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -e +docker run -it --rm iacs/cs107_lecture16:latest diff --git a/lecture16/venv/check_venv.py b/lecture16/venv/check_venv.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# File : check_venv.py +# Description: Check if you run in a virtual Python environment +# Copyright 2022 Harvard University. All Rights Reserved. +import sys + +if __name__ == "__main__": + print(sys.prefix) + print(sys.base_prefix) + try: + assert sys.prefix == sys.base_prefix + except AssertionError as e: + print("--> Virtual Python environment is activated!") diff --git a/lecture17/linked_list.py b/lecture17/linked_list.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# vim: foldmethod=marker +# File : linked_list.py +# Description: Linked list implementation +# Copyright 2022 Harvard University. All Rights Reserved. + +import copy + + +# Node {{{1 +class Node: + def __init__(self, key, *, data=None, next=None): + self.key = key + self.data = data + self.next = next + + +# LinkedList {{{1 +class LinkedList: + # List creation {{{2 + def __init__(self, key, data, *, reverse=False): + """ + Construct a linked list. + + Parameter: + ---------- + key : list-like + List of keys used to identify nodes + data : list-like + List of data objects to be associated with nodes + reverse : bool + Create linked list in reverse order + + """ + start = Node(key[0], data=data[0]) # create root node + end = start + for k, d in zip(key[1:], data[1:]): + if reverse: + end = self._prepend(end, Node(k, data=d)) + else: + end = self._append(end, Node(k, data=d)) + self.root = start if not reverse else end + + @staticmethod + def _append(anchor, node): + anchor.next = node + return node + + @staticmethod + def _prepend(anchor, node): + node.next = anchor + return node + + # Sequence properties {{{2 + def __getitem__(self, key): + _, node = self._getitem(key) + return node + + def __len__(self): + node = self.root + count = 0 + while node != None: + count += 1 + node = node.next + return count + + def _getitem(self, key): + """ + Private method to retrieve a list element given a key. + + Parameters + ---------- + key : + Key for node identification + + Returns + ------- + Tuple with reference to predecessor node and reference to node with + given key + + """ + node = self.root + prev = node + while node != None: + if node.key == key: + return (prev, node) + prev = node + node = node.next + raise KeyError(f"Key '{key}' not found") + + def __str__(self): + node = self.root + pretty = "" + while node != None: + pretty += f"node: {hex(id(node))} : key={node.key} : data={node.data}\n" + node = node.next + return pretty + + # Node insertion {{{2 + def insert(self, key, node, *, before=False): + """Key based node insertion [at worst O(n)].""" + p = self[key] # search for key explicitly -> O(n) + if before: # insert before + tmp = copy.copy(p) # beware! + p.key = node.key + p.data = node.data + node = tmp + p.next = node + else: # insert after + assert p is not None + node.next = p.next + p.next = node + return node + + # Node removal {{{2 + def remove(self, key): + """Key based node removal [at worst O(n)].""" + if self.root.key == key: + del_node = self.root + self.root = self.root.next + else: + # search for key explicitly -> O(n) + prev, del_node = self._getitem(key) + prev.next = del_node.next + del del_node + + # # Iterator {{{2 + # def __iter__(self): + # return LinkedListForwardIterator(self.root) + + +# LinkedListForwardIterator {{{1 +class LinkedListForwardIterator: + def __init__(self, start): + self.node = start # required state to keep track of the iteration + + def __next__(self): + if self.node is None: + raise StopIteration + curr_node = self.node + self.node = self.node.next # modify state of iterator instance + return curr_node + + def __iter__(self): + return self # iter() applied on an iterator must return the iterator itself + + +# main() {{{1 +def main(): + # Create some dummy data class + class Data: + def __init__(self, data): + self.data = data + + def __str__(self): + return f'<value={self.data} {hex(id(self))}>' + + data = [Data(d) for d in list('ABCDE')] + keys = [f'ID_{k}' for k in range(len(data))] + + # create a list + linked_list = LinkedList(keys, data) + print(len(linked_list)) # __len__ + print(linked_list['ID_0']) # __getitem__ (key/index is not an integer!) + + # # Iterate over list {{{2 + # for node in linked_list: + # print(node) + # # 2}}} + + # Insertion / removal examples {{{2 + # insert a new node with different key type + linked_list.insert(keys[1], Node(10, data='Data can be arbitrary')) + + # remove the node again + linked_list.remove(10) + linked_list.remove('ID_0') + # 2}}} + + +if __name__ == "__main__": + main() diff --git a/lecture19/coroutine.py b/lecture19/coroutine.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# File : coroutine.py +# Description: Simple coroutine example +# Copyright 2022 Harvard University. All Rights Reserved. +from inspect import getgeneratorstate + + +def coroutine(): + ncalls = 0 + while True: + x = yield ncalls + ncalls += 1 + print(f'coroutine(): {x} (call id: {ncalls})') + + +def main(): + c = coroutine() + print(getgeneratorstate(c)) + next(c) # prime the coroutine; next() returns 0 here + print(getgeneratorstate(c)) + + c.send('CS107') + print('main(): control back in main function') + n_call = c.send('AC207') + print(f'main(): called coroutine() {n_call} times') + + c.close() + print(getgeneratorstate(c)) + + +if __name__ == "__main__": + main() diff --git a/lecture19/large_data.py b/lecture19/large_data.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# vim: foldmethod=marker +# File : large_data.py +# Description: Example: using generators for reading (large) data files +# Copyright 2022 Harvard University. All Rights Reserved. +import os +import argparse +from itertools import chain + +import numpy as np + + +# argument parser {{{1 +def parse_args(*, partial=False): + parser = argparse.ArgumentParser(description="Generator example with large data.") + # yapf: disable + parser.add_argument('-g', '--generate', action='store_true', + help="Generate the test data (will take up `size` MB of disk space)") + parser.add_argument('-e', '--eager', action='store_true', + help="Include data source obtained with eager reader.") + parser.add_argument('-s', '--size', type=int, default=2048, + help="Size of test data file in MB. Default is 2048MB.") + parser.add_argument('-c', '--chunk_size', type=int, default=1, + help="Chunk size for lazy load of binary data file.") + # yapf: enable + if partial: + return parser.parse_known_args() + else: + return parser.parse_args() + + +# process data {{{1 +def process(*data_iterables): + """ + Process data from multiple sources. + + In this example the data elements are expected to be scalars and are just + summed up. In practice here is where you do the actual work. + + Example: + -------- + Note that data iterables can be iterables, iterators or generators. The + example below uses a list and tuple to demonstrate. + + >>> process([1, 2, 3], (4, 5)) + 15 + + """ + val = 0 + item_count = 0 + for item in chain.from_iterable(data_iterables): + val += item + item_count += 1 + if item_count % 10000 == 0: + print(f'{item_count} items processed from input') + return val + + +# data readers (generators) {{{1 +# lazy binary data reader (generator) {{{2 +def read_binary_lazy(fname, chunk_size=1): + """ + Lazy binary data loader (generator). + + Load data from a binary file specific to an application. This loader is + using a generator to load the data lazy upon request in chunk sizes + specified by `chunk_size`. + + Parameters + ---------- + fname : str + Path to data file. + chunk_size : int + Number of data elements to be loaded from the file. This parameter can + optimize the bandwidth for reading the file. Reading small sizes from a + file is not efficient. + + Yields + ------ + float + One data element from the data file. + + """ + + with open(fname, 'r') as f: + # Assignment expressions `:=` Python 3.8 and beyond only + # (see https://peps.python.org/pep-0572/) + while (chunk := np.fromfile(f, dtype=float, count=chunk_size)).size > 0: + for item in chunk: + yield item + + +# eager binary data reader (iterable) {{{2 +def read_binary_eager(fname): + """ + Eager binary data loader (iterable numpy array). + + Load data from a binary file specific to an application. This loader is + reading all content in the file eagerly. It may consume a large amount of + RAM on your system if the file is large. + + Parameters + ---------- + fname : str + Path to data file. + + Returns + ------- + array_like + NumPy array of data in file. + + """ + with open(fname, 'r') as f: + return np.fromfile(f, dtype=float) + + +# lazy text data reader (generator) {{{2 +def read_ascii_lazy(fname, sep=','): + """ + Lazy ASCII data loader (generator). + + Alternative ASCII data loader for demonstration purpose. Assume you are + working with data sources that deliver the data in either binary form or + ASCII text. This additional data reader allows to read the ASCII version of + the data sources. + + Parameters + ---------- + fname : str + Path to data file. + + Yields + ------ + float + One data element from the data file. + """ + with open(fname, 'r') as f: + # Assignment expressions `:=` Python 3.8 and beyond only + # (see https://peps.python.org/pep-0572/) + while (item := np.fromfile(f, count=1, sep=sep)).size > 0: + yield item + + +# data generation {{{1 +def gen_data(nelements, fname, sep=''): + """ + Generate test data files. + + Parameters + ---------- + nelements : int + Number of random data elements. + fname : str + Path to data file. + sep : str + Data separator. Binary file is generated if empty string. + + """ + x = np.random.rand(nelements) + x.tofile(fname, sep=sep) + + +# main() {{{1 +def main(args): + # generate data {{{2 + if args.generate: + n_floats = args.size * 1024 * 1024 // 8 + gen_data(n_floats, 'data.bin') # random binary data + gen_data(8, 'data.txt', sep=',') # random text data + + # process data {{{2 + if os.path.isfile('data.bin') and os.path.isfile('data.txt'): + sources = [] + + # use an eager reader for demonstration, a large file will occupy a lot + # of RAM + if args.eager: + sources.append(read_binary_eager('data.bin')) + + # performance demonstration is based on the lazy_binary data reader + sources.append(read_binary_lazy('data.bin', args.chunk_size)) + + # ASCII reader is just for additional demonstration that you can easily + # treat different data sources in Python + sources.append(read_ascii_lazy('data.txt')) + + # process all these sources + result = process(*sources) + print('Processed result:', result) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/lecture19/linked_list.py b/lecture19/linked_list.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# vim: foldmethod=marker +# File : linked_list.py +# Description: Linked list example with generator function for __iter__ +# Copyright 2022 Harvard University. All Rights Reserved. +import copy + + +# Node {{{1 +class Node: + def __init__(self, key, *, data=None, next=None): + self.key = key + self.data = data + self.next = next + + +# LinkedList {{{1 +class LinkedList: + # List creation {{{2 + def __init__(self, key, data, *, reverse=False): + start = Node(key[0], data=data[0]) # create root node + end = start + for k, d in zip(key[1:], data[1:]): + if reverse: + end = self._prepend(end, Node(k, data=d)) + else: + end = self._append(end, Node(k, data=d)) + self.first = end if reverse else start + + @staticmethod + def _append(anchor, node): + anchor.next = node + return node + + @staticmethod + def _prepend(anchor, node): + node.next = anchor + return node + + # Sequence properties {{{2 + def __getitem__(self, key): + _, node = self._getitem(key) + return node + + def __len__(self): + node = self.first + count = 0 + while node != None: + count += 1 + node = node.next + return count + + def __str__(self): + node = self.first + pretty = "" + while node != None: + pretty += f"node: {hex(id(node))} : key={node.key} : data={node.data}\n" + node = node.next + return pretty + + def _getitem(self, key): + """Private method to retrieve a list element given a key. + Arguments: + key: identifies the node + Returns: + Tuple with reference to predecessor node and reference to node + with given key + """ + node = self.first + prev = node + while node != None: + if node.key == key: + return (prev, node) + prev = node + node = node.next + raise KeyError(f"Key '{key}' not found") + + # Node insertion {{{2 + def insert(self, key, node, *, before=False): + anchor = self.__getitem__(key) + if before: # insert before + tmp = copy.copy(anchor) # beware! + anchor.key = node.key + anchor.data = node.data + node = tmp + anchor.next = node + else: # insert after + assert anchor is not None + node.next = anchor.next + anchor.next = node + return node + + # Node removal {{{2 + def remove(self, key): + if self.first.key == key: + del_node = self.first + self.first = self.first.next + else: + prev, del_node = self._getitem(key) + prev.next = del_node.next + del del_node + + # Iterator {{{2 + def __iter__(self): + node = self.first + while node != None: + yield node + node = node.next + + +# main() {{{1 +def main(): + # create a list + linked_list = LinkedList(range(5), list('abcde')) + + # iter(ll) returns a generator object + print(type(iter(linked_list))) + for node in linked_list: + print(f'{node} : key={node.key} value={node.data}') + + +if __name__ == "__main__": + main() diff --git a/lecture20/code_object.py b/lecture20/code_object.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import dis + +a = 1 +b = 2 + +# source to be compiled into a code object +source = 'a + 1' +# source = 'a + b' + +# compile code object +co = compile(source, '<string>', mode='eval') + +# disassembly columns are: +# line_number byte_offset instruction_name argument_value +dis.dis(co) + +# names and constants +print(f'names: {co.co_names}') +print(f'constants: {co.co_consts}') + +# opcodes and opargs in binary +print(f'raw binary bytecode: {co.co_code}') +h = co.co_code.hex() +inst_code = " ".join([h[i:i + 4] for i in range(0, len(h), 4)]).split() +print(f'hex representation: {" ".join(inst_code)}') + +for code in inst_code: + opcode = f'0x{code[0:2]}' + oparg = f'0x{code[2:4]}' + print(f'opname lookup: {opcode} -> {dis.opname[int(opcode, 16)]}') diff --git a/lecture20/cpython/3.10.8/ceval.c b/lecture20/cpython/3.10.8/ceval.c @@ -0,0 +1,3045 @@ +// XXX: [fabianw@seas.harvard.edu; 2022-11-08] This code has been shortened to +// fit the context of the lecture. The full code for this file can be found +// here: +// +// https://github.com/python/cpython/blob/v3.10.8/Python/ceval.c + +PyObject* _Py_HOT_FUNCTION +_PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" +#endif + +#ifdef DXPAIRS + int lastopcode = 0; +#endif + PyObject **stack_pointer; /* Next free slot in value stack */ + const _Py_CODEUNIT *next_instr; + int opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ + PyObject **fastlocals, **freevars; + PyObject *retval = NULL; /* Return value */ + _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; + PyCodeObject *co; + + const _Py_CODEUNIT *first_instr; + PyObject *names; + PyObject *consts; + _PyOpcache *co_opcache; + +#ifdef LLTRACE + _Py_IDENTIFIER(__ltrace__); +#endif + + if (_Py_EnterRecursiveCall(tstate, "")) { + return NULL; + } + + PyTraceInfo trace_info; + /* Mark trace_info as uninitialized */ + trace_info.code = NULL; + + /* WARNING: Because the CFrame lives on the C stack, + * but can be accessed from a heap allocated object (tstate) + * strict stack discipline must be maintained. + */ + CFrame *prev_cframe = tstate->cframe; + trace_info.cframe.use_tracing = prev_cframe->use_tracing; + trace_info.cframe.previous = prev_cframe; + tstate->cframe = &trace_info.cframe; + + /* push frame */ + tstate->frame = f; + co = f->f_code; + + if (trace_info.cframe.use_tracing) { + if (tstate->c_tracefunc != NULL) { + /* tstate->c_tracefunc, if defined, is a + function that will be called on *every* entry + to a code block. Its return value, if not + None, is a function that will be called at + the start of each executed line of code. + (Actually, the function must return itself + in order to continue tracing.) The trace + functions are called with three arguments: + a pointer to the current frame, a string + indicating why the function is called, and + an argument which depends on the situation. + The global trace function is also called + whenever an exception is detected. */ + if (call_trace_protected(tstate->c_tracefunc, + tstate->c_traceobj, + tstate, f, &trace_info, + PyTrace_CALL, Py_None)) { + /* Trace function raised an error */ + goto exit_eval_frame; + } + } + if (tstate->c_profilefunc != NULL) { + /* Similar for c_profilefunc, except it needn't + return itself and isn't called for "line" events */ + if (call_trace_protected(tstate->c_profilefunc, + tstate->c_profileobj, + tstate, f, &trace_info, + PyTrace_CALL, Py_None)) { + /* Profile function raised an error */ + goto exit_eval_frame; + } + } + } + + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) + dtrace_function_entry(f); + + names = co->co_names; + consts = co->co_consts; + fastlocals = f->f_localsplus; + freevars = f->f_localsplus + co->co_nlocals; + assert(PyBytes_Check(co->co_code)); + assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX); + assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0); + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(co->co_code), sizeof(_Py_CODEUNIT))); + first_instr = (_Py_CODEUNIT *) PyBytes_AS_STRING(co->co_code); + /* + f->f_lasti refers to the index of the last instruction, + unless it's -1 in which case next_instr should be first_instr. + + YIELD_FROM sets f_lasti to itself, in order to repeatedly yield + multiple values. + + When the PREDICT() macros are enabled, some opcode pairs follow in + direct succession without updating f->f_lasti. A successful + prediction effectively links the two codes together as if they + were a single new opcode; accordingly,f->f_lasti will point to + the first code in the pair (for instance, GET_ITER followed by + FOR_ITER is effectively a single opcode and f->f_lasti will point + to the beginning of the combined pair.) + */ + assert(f->f_lasti >= -1); + next_instr = first_instr + f->f_lasti + 1; + stack_pointer = f->f_valuestack + f->f_stackdepth; + /* Set f->f_stackdepth to -1. + * Update when returning or calling trace function. + Having f_stackdepth <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. + */ + f->f_stackdepth = -1; + f->f_state = FRAME_EXECUTING; + + if (co->co_opcache_flag < opcache_min_runs) { + co->co_opcache_flag++; + if (co->co_opcache_flag == opcache_min_runs) { + if (_PyCode_InitOpcache(co) < 0) { + goto exit_eval_frame; + } +#if OPCACHE_STATS + opcache_code_objects_extra_mem += + PyBytes_Size(co->co_code) / sizeof(_Py_CODEUNIT) + + sizeof(_PyOpcache) * co->co_opcache_size; + opcache_code_objects++; +#endif + } + } + +#ifdef LLTRACE + { + int r = _PyDict_ContainsId(f->f_globals, &PyId___ltrace__); + if (r < 0) { + goto exit_eval_frame; + } + lltrace = r; + } +#endif + + if (throwflag) { /* support for generator.throw() */ + goto error; + } + +#ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif + +main_loop: + for (;;) { + assert(stack_pointer >= f->f_valuestack); /* else underflow */ + assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */ + assert(!_PyErr_Occurred(tstate)); + + /* Do periodic things. Doing this every time through + the loop would add too much overhead, so we do it + only every Nth instruction. We also do it if + ``pending.calls_to_do'' is set, i.e. when an asynchronous + event needs attention (e.g. a signal handler or + async I/O handler); see Py_AddPendingCall() and + Py_MakePendingCalls() above. */ + + if (_Py_atomic_load_relaxed(eval_breaker)) { + opcode = _Py_OPCODE(*next_instr); + if (opcode != SETUP_FINALLY && + opcode != SETUP_WITH && + opcode != BEFORE_ASYNC_WITH && + opcode != YIELD_FROM) { + /* Few cases where we skip running signal handlers and other + pending calls: + - If we're about to enter the 'with:'. It will prevent + emitting a resource warning in the common idiom + 'with open(path) as file:'. + - If we're about to enter the 'async with:'. + - If we're about to enter the 'try:' of a try/finally (not + *very* useful, but might help in some cases and it's + traditional) + - If we're resuming a chain of nested 'yield from' or + 'await' calls, then each frame is parked with YIELD_FROM + as its next opcode. If the user hit control-C we want to + wait until we've reached the innermost frame before + running the signal handler and raising KeyboardInterrupt + (see bpo-30039). + */ + if (eval_frame_handle_pending(tstate) != 0) { + goto error; + } + } + } + + tracing_dispatch: + { + int instr_prev = f->f_lasti; + f->f_lasti = INSTR_OFFSET(); + NEXTOPARG(); + + if (PyDTrace_LINE_ENABLED()) + maybe_dtrace_line(f, &trace_info, instr_prev); + + /* line-by-line tracing support */ + + if (trace_info.cframe.use_tracing && + tstate->c_tracefunc != NULL && !tstate->tracing) { + int err; + /* see maybe_call_line_trace() + for expository comments */ + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); + + err = maybe_call_line_trace(tstate->c_tracefunc, + tstate->c_traceobj, + tstate, f, + &trace_info, instr_prev); + /* Reload possibly changed frame fields */ + JUMPTO(f->f_lasti); + stack_pointer = f->f_valuestack+f->f_stackdepth; + f->f_stackdepth = -1; + if (err) { + /* trace function raised an exception */ + goto error; + } + NEXTOPARG(); + } + } + +#ifdef LLTRACE + /* Instruction tracing */ + + if (lltrace) { + if (HAS_ARG(opcode)) { + printf("%d: %d, %d\n", + f->f_lasti, opcode, oparg); + } + else { + printf("%d: %d\n", + f->f_lasti, opcode); + } + } +#endif +#if USE_COMPUTED_GOTOS == 0 + goto dispatch_opcode; + + predispatch: + if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) { + goto tracing_dispatch; + } + f->f_lasti = INSTR_OFFSET(); + NEXTOPARG(); +#endif + dispatch_opcode: +#ifdef DYNAMIC_EXECUTION_PROFILE +#ifdef DXPAIRS + dxpairs[lastopcode][opcode]++; + lastopcode = opcode; +#endif + dxp[opcode]++; +#endif + + switch (opcode) { + + /* BEWARE! + It is essential that any operation that fails must goto error + and that all operation that succeed call DISPATCH() ! */ + + case TARGET(NOP): { + DISPATCH(); + } + + case TARGET(LOAD_FAST): { + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + format_exc_check_arg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(co->co_varnames, oparg)); + goto error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + case TARGET(LOAD_CONST): { + PREDICTED(LOAD_CONST); + PyObject *value = GETITEM(consts, oparg); + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + case TARGET(STORE_FAST): { + PREDICTED(STORE_FAST); + PyObject *value = POP(); + SETLOCAL(oparg, value); + DISPATCH(); + } + + case TARGET(POP_TOP): { + PyObject *value = POP(); + Py_DECREF(value); + DISPATCH(); + } + + case TARGET(ROT_TWO): { + PyObject *top = TOP(); + PyObject *second = SECOND(); + SET_TOP(second); + SET_SECOND(top); + DISPATCH(); + } + + case TARGET(ROT_THREE): { + PyObject *top = TOP(); + PyObject *second = SECOND(); + PyObject *third = THIRD(); + SET_TOP(second); + SET_SECOND(third); + SET_THIRD(top); + DISPATCH(); + } + + case TARGET(ROT_FOUR): { + PyObject *top = TOP(); + PyObject *second = SECOND(); + PyObject *third = THIRD(); + PyObject *fourth = FOURTH(); + SET_TOP(second); + SET_SECOND(third); + SET_THIRD(fourth); + SET_FOURTH(top); + DISPATCH(); + } + + case TARGET(DUP_TOP): { + PyObject *top = TOP(); + Py_INCREF(top); + PUSH(top); + DISPATCH(); + } + + case TARGET(DUP_TOP_TWO): { + PyObject *top = TOP(); + PyObject *second = SECOND(); + Py_INCREF(top); + Py_INCREF(second); + STACK_GROW(2); + SET_TOP(top); + SET_SECOND(second); + DISPATCH(); + } + + case TARGET(UNARY_POSITIVE): { + PyObject *value = TOP(); + PyObject *res = PyNumber_Positive(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(UNARY_NEGATIVE): { + PyObject *value = TOP(); + PyObject *res = PyNumber_Negative(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(UNARY_NOT): { + PyObject *value = TOP(); + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err == 0) { + Py_INCREF(Py_True); + SET_TOP(Py_True); + DISPATCH(); + } + else if (err > 0) { + Py_INCREF(Py_False); + SET_TOP(Py_False); + DISPATCH(); + } + STACK_SHRINK(1); + goto error; + } + + case TARGET(UNARY_INVERT): { + PyObject *value = TOP(); + PyObject *res = PyNumber_Invert(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_POWER): { + PyObject *exp = POP(); + PyObject *base = TOP(); + PyObject *res = PyNumber_Power(base, exp, Py_None); + Py_DECREF(base); + Py_DECREF(exp); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_MULTIPLY): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_Multiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_MATRIX_MULTIPLY): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_MatrixMultiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_TRUE_DIVIDE): { + PyObject *divisor = POP(); + PyObject *dividend = TOP(); + PyObject *quotient = PyNumber_TrueDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + SET_TOP(quotient); + if (quotient == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_FLOOR_DIVIDE): { + PyObject *divisor = POP(); + PyObject *dividend = TOP(); + PyObject *quotient = PyNumber_FloorDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + SET_TOP(quotient); + if (quotient == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_MODULO): { + PyObject *divisor = POP(); + PyObject *dividend = TOP(); + PyObject *res; + if (PyUnicode_CheckExact(dividend) && ( + !PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { + // fast path; string formatting, but not if the RHS is a str subclass + // (see issue28598) + res = PyUnicode_Format(dividend, divisor); + } else { + res = PyNumber_Remainder(dividend, divisor); + } + Py_DECREF(divisor); + Py_DECREF(dividend); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_ADD): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *sum; + /* NOTE(vstinner): Please don't try to micro-optimize int+int on + CPython using bytecode, it is simply worthless. + See http://bugs.python.org/issue21955 and + http://bugs.python.org/issue10044 for the discussion. In short, + no patch shown any impact on a realistic benchmark, only a minor + speedup on microbenchmarks. */ + if (PyUnicode_CheckExact(left) && + PyUnicode_CheckExact(right)) { + sum = unicode_concatenate(tstate, left, right, f, next_instr); + /* unicode_concatenate consumed the ref to left */ + } + else { + sum = PyNumber_Add(left, right); + Py_DECREF(left); + } + Py_DECREF(right); + SET_TOP(sum); + if (sum == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_SUBTRACT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *diff = PyNumber_Subtract(left, right); + Py_DECREF(right); + Py_DECREF(left); + SET_TOP(diff); + if (diff == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_SUBSCR): { + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_LSHIFT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_Lshift(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_RSHIFT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_Rshift(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_AND): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_And(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_XOR): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_Xor(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(BINARY_OR): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_Or(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(LIST_APPEND): { + PyObject *v = POP(); + PyObject *list = PEEK(oparg); + int err; + err = PyList_Append(list, v); + Py_DECREF(v); + if (err != 0) + goto error; + PREDICT(JUMP_ABSOLUTE); + DISPATCH(); + } + + case TARGET(SET_ADD): { + PyObject *v = POP(); + PyObject *set = PEEK(oparg); + int err; + err = PySet_Add(set, v); + Py_DECREF(v); + if (err != 0) + goto error; + PREDICT(JUMP_ABSOLUTE); + DISPATCH(); + } + + case TARGET(INPLACE_POWER): { + PyObject *exp = POP(); + PyObject *base = TOP(); + PyObject *res = PyNumber_InPlacePower(base, exp, Py_None); + Py_DECREF(base); + Py_DECREF(exp); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_MULTIPLY): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceMultiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_MATRIX_MULTIPLY): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceMatrixMultiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_TRUE_DIVIDE): { + PyObject *divisor = POP(); + PyObject *dividend = TOP(); + PyObject *quotient = PyNumber_InPlaceTrueDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + SET_TOP(quotient); + if (quotient == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_FLOOR_DIVIDE): { + PyObject *divisor = POP(); + PyObject *dividend = TOP(); + PyObject *quotient = PyNumber_InPlaceFloorDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + SET_TOP(quotient); + if (quotient == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_MODULO): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *mod = PyNumber_InPlaceRemainder(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(mod); + if (mod == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_ADD): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *sum; + if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { + sum = unicode_concatenate(tstate, left, right, f, next_instr); + /* unicode_concatenate consumed the ref to left */ + } + else { + sum = PyNumber_InPlaceAdd(left, right); + Py_DECREF(left); + } + Py_DECREF(right); + SET_TOP(sum); + if (sum == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_SUBTRACT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *diff = PyNumber_InPlaceSubtract(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(diff); + if (diff == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_LSHIFT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceLshift(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_RSHIFT): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceRshift(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_AND): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceAnd(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_XOR): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceXor(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(INPLACE_OR): { + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyNumber_InPlaceOr(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(STORE_SUBSCR): { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + PyObject *v = THIRD(); + int err; + STACK_SHRINK(3); + /* container[sub] = v */ + err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(DELETE_SUBSCR): { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + int err; + STACK_SHRINK(2); + /* del container[sub] */ + err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(PRINT_EXPR): { + _Py_IDENTIFIER(displayhook); + PyObject *value = POP(); + PyObject *hook = _PySys_GetObjectId(&PyId_displayhook); + PyObject *res; + if (hook == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "lost sys.displayhook"); + Py_DECREF(value); + goto error; + } + res = PyObject_CallOneArg(hook, value); + Py_DECREF(value); + if (res == NULL) + goto error; + Py_DECREF(res); + DISPATCH(); + } + + case TARGET(RAISE_VARARGS): { + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = POP(); /* cause */ + /* fall through */ + case 1: + exc = POP(); /* exc */ + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; + } + goto error; + } + + case TARGET(RETURN_VALUE): { + retval = POP(); + assert(f->f_iblock == 0); + assert(EMPTY()); + f->f_state = FRAME_RETURNED; + f->f_stackdepth = 0; + goto exiting; + } + + case TARGET(GET_AITER): { + unaryfunc getter = NULL; + PyObject *iter = NULL; + PyObject *obj = TOP(); + PyTypeObject *type = Py_TYPE(obj); + + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + + if (getter != NULL) { + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) { + SET_TOP(NULL); + goto error; + } + } + else { + SET_TOP(NULL); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + goto error; + } + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + + SET_TOP(NULL); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + goto error; + } + + SET_TOP(iter); + DISPATCH(); + } + + case TARGET(GET_ANEXT): { + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyObject *awaitable = NULL; + PyObject *aiter = TOP(); + PyTypeObject *type = Py_TYPE(aiter); + + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + goto error; + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + goto error; + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + goto error; + } + + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } + } + + PUSH(awaitable); + PREDICT(LOAD_CONST); + DISPATCH(); + } + + case TARGET(GET_AWAITABLE): { + PREDICTED(GET_AWAITABLE); + PyObject *iterable = TOP(); + PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + + if (iter == NULL) { + int opcode_at_minus_3 = 0; + if ((next_instr - first_instr) > 2) { + opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]); + } + format_awaitable_error(tstate, Py_TYPE(iterable), + opcode_at_minus_3, + _Py_OPCODE(next_instr[-2])); + } + + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } + } + + SET_TOP(iter); /* Even if it's NULL */ + + if (iter == NULL) { + goto error; + } + + PREDICT(LOAD_CONST); + DISPATCH(); + } + + case TARGET(YIELD_FROM): { + PyObject *v = POP(); + PyObject *receiver = TOP(); + PySendResult gen_status; + if (tstate->c_tracefunc == NULL) { + gen_status = PyIter_Send(receiver, v, &retval); + } else { + _Py_IDENTIFIER(send); + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = _PyObject_CallMethodIdOneArg(receiver, &PyId_send, v); + } + if (retval == NULL) { + if (tstate->c_tracefunc != NULL + && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + gen_status = PYGEN_RETURN; + } + else { + gen_status = PYGEN_ERROR; + } + } + else { + gen_status = PYGEN_NEXT; + } + } + Py_DECREF(v); + if (gen_status == PYGEN_ERROR) { + assert (retval == NULL); + goto error; + } + if (gen_status == PYGEN_RETURN) { + assert (retval != NULL); + + Py_DECREF(receiver); + SET_TOP(retval); + retval = NULL; + DISPATCH(); + } + assert (gen_status == PYGEN_NEXT); + /* receiver remains on stack, retval is value to be yielded */ + /* and repeat... */ + assert(f->f_lasti > 0); + f->f_lasti -= 1; + f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); + goto exiting; + } + + case TARGET(YIELD_VALUE): { + retval = POP(); + + if (co->co_flags & CO_ASYNC_GENERATOR) { + PyObject *w = _PyAsyncGenValueWrapperNew(retval); + Py_DECREF(retval); + if (w == NULL) { + retval = NULL; + goto error; + } + retval = w; + } + f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); + goto exiting; + } + + case TARGET(GEN_START): { + PyObject *none = POP(); + assert(none == Py_None); + assert(oparg < 3); + Py_DECREF(none); + DISPATCH(); + } + + case TARGET(POP_EXCEPT): { + PyObject *type, *value, *traceback; + _PyErr_StackItem *exc_info; + PyTryBlock *b = PyFrame_BlockPop(f); + if (b->b_type != EXCEPT_HANDLER) { + _PyErr_SetString(tstate, PyExc_SystemError, + "popped block is not an except handler"); + goto error; + } + assert(STACK_LEVEL() >= (b)->b_level + 3 && + STACK_LEVEL() <= (b)->b_level + 4); + exc_info = tstate->exc_info; + type = exc_info->exc_type; + value = exc_info->exc_value; + traceback = exc_info->exc_traceback; + exc_info->exc_type = POP(); + exc_info->exc_value = POP(); + exc_info->exc_traceback = POP(); + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + DISPATCH(); + } + + case TARGET(POP_BLOCK): { + PyFrame_BlockPop(f); + DISPATCH(); + } + + case TARGET(RERAISE): { + assert(f->f_iblock > 0); + if (oparg) { + f->f_lasti = f->f_blockstack[f->f_iblock-1].b_handler; + } + PyObject *exc = POP(); + PyObject *val = POP(); + PyObject *tb = POP(); + assert(PyExceptionClass_Check(exc)); + _PyErr_Restore(tstate, exc, val, tb); + goto exception_unwind; + } + + case TARGET(END_ASYNC_FOR): { + PyObject *exc = POP(); + assert(PyExceptionClass_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + PyTryBlock *b = PyFrame_BlockPop(f); + assert(b->b_type == EXCEPT_HANDLER); + Py_DECREF(exc); + UNWIND_EXCEPT_HANDLER(b); + Py_DECREF(POP()); + JUMPBY(oparg); + DISPATCH(); + } + else { + PyObject *val = POP(); + PyObject *tb = POP(); + _PyErr_Restore(tstate, exc, val, tb); + goto exception_unwind; + } + } + + case TARGET(LOAD_ASSERTION_ERROR): { + PyObject *value = PyExc_AssertionError; + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + case TARGET(LOAD_BUILD_CLASS): { + _Py_IDENTIFIER(__build_class__); + + PyObject *bc; + if (PyDict_CheckExact(f->f_builtins)) { + bc = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___build_class__); + if (bc == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + } + goto error; + } + Py_INCREF(bc); + } + else { + PyObject *build_class_str = _PyUnicode_FromId(&PyId___build_class__); + if (build_class_str == NULL) + goto error; + bc = PyObject_GetItem(f->f_builtins, build_class_str); + if (bc == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + goto error; + } + } + PUSH(bc); + DISPATCH(); + } + + case TARGET(STORE_NAME): { + PyObject *name = GETITEM(names, oparg); + PyObject *v = POP(); + PyObject *ns = f->f_locals; + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + goto error; + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(DELETE_NAME): { + PyObject *name = GETITEM(names, oparg); + PyObject *ns = f->f_locals; + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + goto error; + } + err = PyObject_DelItem(ns, name); + if (err != 0) { + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + goto error; + } + DISPATCH(); + } + + case TARGET(UNPACK_SEQUENCE): { + PREDICTED(UNPACK_SEQUENCE); + PyObject *seq = POP(), *item, **items; + if (PyTuple_CheckExact(seq) && + PyTuple_GET_SIZE(seq) == oparg) { + items = ((PyTupleObject *)seq)->ob_item; + while (oparg--) { + item = items[oparg]; + Py_INCREF(item); + PUSH(item); + } + } else if (PyList_CheckExact(seq) && + PyList_GET_SIZE(seq) == oparg) { + items = ((PyListObject *)seq)->ob_item; + while (oparg--) { + item = items[oparg]; + Py_INCREF(item); + PUSH(item); + } + } else if (unpack_iterable(tstate, seq, oparg, -1, + stack_pointer + oparg)) { + STACK_GROW(oparg); + } else { + /* unpack_iterable() raised an exception */ + Py_DECREF(seq); + goto error; + } + Py_DECREF(seq); + DISPATCH(); + } + + case TARGET(UNPACK_EX): { + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject *seq = POP(); + + if (unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, + stack_pointer + totalargs)) { + stack_pointer += totalargs; + } else { + Py_DECREF(seq); + goto error; + } + Py_DECREF(seq); + DISPATCH(); + } + + case TARGET(STORE_ATTR): { + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(DELETE_ATTR): { + PyObject *name = GETITEM(names, oparg); + PyObject *owner = POP(); + int err; + err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + Py_DECREF(owner); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(STORE_GLOBAL): { + PyObject *name = GETITEM(names, oparg); + PyObject *v = POP(); + int err; + err = PyDict_SetItem(f->f_globals, name, v); + Py_DECREF(v); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(DELETE_GLOBAL): { + PyObject *name = GETITEM(names, oparg); + int err; + err = PyDict_DelItem(f->f_globals, name); + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + DISPATCH(); + } + + case TARGET(LOAD_NAME): { + PyObject *name = GETITEM(names, oparg); + PyObject *locals = f->f_locals; + PyObject *v; + if (locals == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when loading %R", name); + goto error; + } + if (PyDict_CheckExact(locals)) { + v = PyDict_GetItemWithError(locals, name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + } + else { + v = PyObject_GetItem(locals, name); + if (v == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) + goto error; + _PyErr_Clear(tstate); + } + } + if (v == NULL) { + v = PyDict_GetItemWithError(f->f_globals, name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyDict_CheckExact(f->f_builtins)) { + v = PyDict_GetItemWithError(f->f_builtins, name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + Py_INCREF(v); + } + else { + v = PyObject_GetItem(f->f_builtins, name); + if (v == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + } + } + } + PUSH(v); + DISPATCH(); + } + + case TARGET(LOAD_GLOBAL): { + PyObject *name; + PyObject *v; + if (PyDict_CheckExact(f->f_globals) + && PyDict_CheckExact(f->f_builtins)) + { + OPCACHE_CHECK(); + if (co_opcache != NULL && co_opcache->optimized > 0) { + _PyOpcache_LoadGlobal *lg = &co_opcache->u.lg; + + if (lg->globals_ver == + ((PyDictObject *)f->f_globals)->ma_version_tag + && lg->builtins_ver == + ((PyDictObject *)f->f_builtins)->ma_version_tag) + { + PyObject *ptr = lg->ptr; + OPCACHE_STAT_GLOBAL_HIT(); + assert(ptr != NULL); + Py_INCREF(ptr); + PUSH(ptr); + DISPATCH(); + } + } + + name = GETITEM(names, oparg); + v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals, + (PyDictObject *)f->f_builtins, + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + + if (co_opcache != NULL) { + _PyOpcache_LoadGlobal *lg = &co_opcache->u.lg; + + if (co_opcache->optimized == 0) { + /* Wasn't optimized before. */ + OPCACHE_STAT_GLOBAL_OPT(); + } else { + OPCACHE_STAT_GLOBAL_MISS(); + } + + co_opcache->optimized = 1; + lg->globals_ver = + ((PyDictObject *)f->f_globals)->ma_version_tag; + lg->builtins_ver = + ((PyDictObject *)f->f_builtins)->ma_version_tag; + lg->ptr = v; /* borrowed */ + } + + Py_INCREF(v); + } + else { + /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ + name = GETITEM(names, oparg); + v = PyObject_GetItem(f->f_globals, name); + if (v == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + + /* namespace 2: builtins */ + v = PyObject_GetItem(f->f_builtins, name); + if (v == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + } + } + PUSH(v); + DISPATCH(); + } + + case TARGET(DELETE_FAST): { + PyObject *v = GETLOCAL(oparg); + if (v != NULL) { + SETLOCAL(oparg, NULL); + DISPATCH(); + } + format_exc_check_arg( + tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(co->co_varnames, oparg) + ); + goto error; + } + + case TARGET(DELETE_DEREF): { + PyObject *cell = freevars[oparg]; + PyObject *oldobj = PyCell_GET(cell); + if (oldobj != NULL) { + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); + DISPATCH(); + } + format_exc_unbound(tstate, co, oparg); + goto error; + } + + case TARGET(LOAD_CLOSURE): { + PyObject *cell = freevars[oparg]; + Py_INCREF(cell); + PUSH(cell); + DISPATCH(); + } + + case TARGET(LOAD_CLASSDEREF): { + PyObject *name, *value, *locals = f->f_locals; + Py_ssize_t idx; + assert(locals); + assert(oparg >= PyTuple_GET_SIZE(co->co_cellvars)); + idx = oparg - PyTuple_GET_SIZE(co->co_cellvars); + assert(idx >= 0 && idx < PyTuple_GET_SIZE(co->co_freevars)); + name = PyTuple_GET_ITEM(co->co_freevars, idx); + if (PyDict_CheckExact(locals)) { + value = PyDict_GetItemWithError(locals, name); + if (value != NULL) { + Py_INCREF(value); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + } + else { + value = PyObject_GetItem(locals, name); + if (value == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + } + } + if (!value) { + PyObject *cell = freevars[oparg]; + value = PyCell_GET(cell); + if (value == NULL) { + format_exc_unbound(tstate, co, oparg); + goto error; + } + Py_INCREF(value); + } + PUSH(value); + DISPATCH(); + } + + case TARGET(LOAD_DEREF): { + PyObject *cell = freevars[oparg]; + PyObject *value = PyCell_GET(cell); + if (value == NULL) { + format_exc_unbound(tstate, co, oparg); + goto error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + case TARGET(STORE_DEREF): { + PyObject *v = POP(); + PyObject *cell = freevars[oparg]; + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + DISPATCH(); + } + + case TARGET(BUILD_STRING): { + PyObject *str; + PyObject *empty = PyUnicode_New(0, 0); + if (empty == NULL) { + goto error; + } + str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); + Py_DECREF(empty); + if (str == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + Py_DECREF(item); + } + PUSH(str); + DISPATCH(); + } + + case TARGET(BUILD_TUPLE): { + PyObject *tup = PyTuple_New(oparg); + if (tup == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + PyTuple_SET_ITEM(tup, oparg, item); + } + PUSH(tup); + DISPATCH(); + } + + case TARGET(BUILD_LIST): { + PyObject *list = PyList_New(oparg); + if (list == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + PyList_SET_ITEM(list, oparg, item); + } + PUSH(list); + DISPATCH(); + } + + case TARGET(LIST_TO_TUPLE): { + PyObject *list = POP(); + PyObject *tuple = PyList_AsTuple(list); + Py_DECREF(list); + if (tuple == NULL) { + goto error; + } + PUSH(tuple); + DISPATCH(); + } + + case TARGET(LIST_EXTEND): { + PyObject *iterable = POP(); + PyObject *list = PEEK(oparg); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + goto error; + } + Py_DECREF(none_val); + Py_DECREF(iterable); + DISPATCH(); + } + + case TARGET(SET_UPDATE): { + PyObject *iterable = POP(); + PyObject *set = PEEK(oparg); + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) { + goto error; + } + DISPATCH(); + } + + case TARGET(BUILD_SET): { + PyObject *set = PySet_New(NULL); + int err = 0; + int i; + if (set == NULL) + goto error; + for (i = oparg; i > 0; i--) { + PyObject *item = PEEK(i); + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + STACK_SHRINK(oparg); + if (err != 0) { + Py_DECREF(set); + goto error; + } + PUSH(set); + DISPATCH(); + } + + case TARGET(BUILD_MAP): { + Py_ssize_t i; + PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg); + if (map == NULL) + goto error; + for (i = oparg; i > 0; i--) { + int err; + PyObject *key = PEEK(2*i); + PyObject *value = PEEK(2*i - 1); + err = PyDict_SetItem(map, key, value); + if (err != 0) { + Py_DECREF(map); + goto error; + } + } + + while (oparg--) { + Py_DECREF(POP()); + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + case TARGET(SETUP_ANNOTATIONS): { + _Py_IDENTIFIER(__annotations__); + int err; + PyObject *ann_dict; + if (f->f_locals == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + goto error; + } + /* check if __annotations__ in locals()... */ + if (PyDict_CheckExact(f->f_locals)) { + ann_dict = _PyDict_GetItemIdWithError(f->f_locals, + &PyId___annotations__); + if (ann_dict == NULL) { + if (_PyErr_Occurred(tstate)) { + goto error; + } + /* ...if not, create a new one */ + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = _PyDict_SetItemId(f->f_locals, + &PyId___annotations__, ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { + goto error; + } + } + } + else { + /* do the same if locals() is not a dict */ + PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__); + if (ann_str == NULL) { + goto error; + } + ann_dict = PyObject_GetItem(f->f_locals, ann_str); + if (ann_dict == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = PyObject_SetItem(f->f_locals, ann_str, ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { + goto error; + } + } + else { + Py_DECREF(ann_dict); + } + } + DISPATCH(); + } + + case TARGET(BUILD_CONST_KEY_MAP): { + Py_ssize_t i; + PyObject *map; + PyObject *keys = TOP(); + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + goto error; + } + map = _PyDict_NewPresized((Py_ssize_t)oparg); + if (map == NULL) { + goto error; + } + for (i = oparg; i > 0; i--) { + int err; + PyObject *key = PyTuple_GET_ITEM(keys, oparg - i); + PyObject *value = PEEK(i + 1); + err = PyDict_SetItem(map, key, value); + if (err != 0) { + Py_DECREF(map); + goto error; + } + } + + Py_DECREF(POP()); + while (oparg--) { + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + case TARGET(DICT_UPDATE): { + PyObject *update = POP(); + PyObject *dict = PEEK(oparg); + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + goto error; + } + Py_DECREF(update); + DISPATCH(); + } + + case TARGET(DICT_MERGE): { + PyObject *update = POP(); + PyObject *dict = PEEK(oparg); + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + format_kwargs_error(tstate, PEEK(2 + oparg), update); + Py_DECREF(update); + goto error; + } + Py_DECREF(update); + PREDICT(CALL_FUNCTION_EX); + DISPATCH(); + } + + case TARGET(MAP_ADD): { + PyObject *value = TOP(); + PyObject *key = SECOND(); + PyObject *map; + int err; + STACK_SHRINK(2); + map = PEEK(oparg); /* dict */ + assert(PyDict_CheckExact(map)); + err = PyDict_SetItem(map, key, value); /* map[key] = value */ + Py_DECREF(value); + Py_DECREF(key); + if (err != 0) + goto error; + PREDICT(JUMP_ABSOLUTE); + DISPATCH(); + } + + case TARGET(LOAD_ATTR): { + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + + PyTypeObject *type = Py_TYPE(owner); + PyObject *res; + PyObject **dictptr; + PyObject *dict; + _PyOpCodeOpt_LoadAttr *la; + + OPCACHE_STAT_ATTR_TOTAL(); + + OPCACHE_CHECK(); + if (co_opcache != NULL && PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) + { + if (co_opcache->optimized > 0) { + // Fast path -- cache hit makes LOAD_ATTR ~30% faster. + la = &co_opcache->u.la; + if (la->type == type && la->tp_version_tag == type->tp_version_tag) + { + // Hint >= 0 is a dict index; hint == -1 is a dict miss. + // Hint < -1 is an inverted slot offset: offset is strictly > 0, + // so ~offset is strictly < -1 (assuming 2's complement). + if (la->hint < -1) { + // Even faster path -- slot hint. + Py_ssize_t offset = ~la->hint; + // fprintf(stderr, "Using hint for offset %zd\n", offset); + char *addr = (char *)owner + offset; + res = *(PyObject **)addr; + if (res != NULL) { + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + DISPATCH(); + } + // Else slot is NULL. Fall through to slow path to raise AttributeError(name). + // Don't DEOPT, since the slot is still there. + } else { + // Fast path for dict. + assert(type->tp_dict != NULL); + assert(type->tp_dictoffset > 0); + + dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); + dict = *dictptr; + if (dict != NULL && PyDict_CheckExact(dict)) { + Py_ssize_t hint = la->hint; + Py_INCREF(dict); + res = NULL; + assert(!_PyErr_Occurred(tstate)); + la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res); + if (res != NULL) { + assert(la->hint >= 0); + if (la->hint == hint && hint >= 0) { + // Our hint has helped -- cache hit. + OPCACHE_STAT_ATTR_HIT(); + } else { + // The hint we provided didn't work. + // Maybe next time? + OPCACHE_MAYBE_DEOPT_LOAD_ATTR(); + } + + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + Py_DECREF(dict); + DISPATCH(); + } + else { + _PyErr_Clear(tstate); + // This attribute can be missing sometimes; + // we don't want to optimize this lookup. + OPCACHE_DEOPT_LOAD_ATTR(); + Py_DECREF(dict); + } + } + else { + // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } + } + else { + // The type of the object has either been updated, + // or is different. Maybe it will stabilize? + OPCACHE_MAYBE_DEOPT_LOAD_ATTR(); + } + OPCACHE_STAT_ATTR_MISS(); + } + + if (co_opcache != NULL && // co_opcache can be NULL after a DEOPT() call. + type->tp_getattro == PyObject_GenericGetAttr) + { + if (type->tp_dict == NULL) { + if (PyType_Ready(type) < 0) { + Py_DECREF(owner); + SET_TOP(NULL); + goto error; + } + } + PyObject *descr = _PyType_Lookup(type, name); + if (descr != NULL) { + // We found an attribute with a data-like descriptor. + PyTypeObject *dtype = Py_TYPE(descr); + if (dtype == &PyMemberDescr_Type) { // It's a slot + PyMemberDescrObject *member = (PyMemberDescrObject *)descr; + struct PyMemberDef *dmem = member->d_member; + if (dmem->type == T_OBJECT_EX) { + Py_ssize_t offset = dmem->offset; + assert(offset > 0); // 0 would be confused with dict hint == -1 (miss). + + if (co_opcache->optimized == 0) { + // First time we optimize this opcode. + OPCACHE_STAT_ATTR_OPT(); + co_opcache->optimized = OPCODE_CACHE_MAX_TRIES; + // fprintf(stderr, "Setting hint for %s, offset %zd\n", dmem->name, offset); + } + + la = &co_opcache->u.la; + la->type = type; + la->tp_version_tag = type->tp_version_tag; + la->hint = ~offset; + + char *addr = (char *)owner + offset; + res = *(PyObject **)addr; + if (res != NULL) { + Py_INCREF(res); + Py_DECREF(owner); + SET_TOP(res); + + DISPATCH(); + } + // Else slot is NULL. Fall through to slow path to raise AttributeError(name). + } + // Else it's a slot of a different type. We don't handle those. + } + // Else it's some other kind of descriptor that we don't handle. + OPCACHE_DEOPT_LOAD_ATTR(); + } + else if (type->tp_dictoffset > 0) { + // We found an instance with a __dict__. + dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); + dict = *dictptr; + + if (dict != NULL && PyDict_CheckExact(dict)) { + Py_INCREF(dict); + res = NULL; + assert(!_PyErr_Occurred(tstate)); + Py_ssize_t hint = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res); + if (res != NULL) { + Py_INCREF(res); + Py_DECREF(dict); + Py_DECREF(owner); + SET_TOP(res); + + if (co_opcache->optimized == 0) { + // First time we optimize this opcode. + OPCACHE_STAT_ATTR_OPT(); + co_opcache->optimized = OPCODE_CACHE_MAX_TRIES; + } + + la = &co_opcache->u.la; + la->type = type; + la->tp_version_tag = type->tp_version_tag; + assert(hint >= 0); + la->hint = hint; + + DISPATCH(); + } + else { + _PyErr_Clear(tstate); + } + Py_DECREF(dict); + } else { + // There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } else { + // The object's class does not have a tp_dictoffset we can use. + OPCACHE_DEOPT_LOAD_ATTR(); + } + } else if (type->tp_getattro != PyObject_GenericGetAttr) { + OPCACHE_DEOPT_LOAD_ATTR(); + } + } + + // Slow path. + res = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(COMPARE_OP): { + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) + goto error; + PREDICT(POP_JUMP_IF_FALSE); + PREDICT(POP_JUMP_IF_TRUE); + DISPATCH(); + } + + case TARGET(IS_OP): { + PyObject *right = POP(); + PyObject *left = TOP(); + int res = Py_Is(left, right) ^ oparg; + PyObject *b = res ? Py_True : Py_False; + Py_INCREF(b); + SET_TOP(b); + Py_DECREF(left); + Py_DECREF(right); + PREDICT(POP_JUMP_IF_FALSE); + PREDICT(POP_JUMP_IF_TRUE); + DISPATCH(); + } + + case TARGET(CONTAINS_OP): { + PyObject *right = POP(); + PyObject *left = POP(); + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) { + goto error; + } + PyObject *b = (res^oparg) ? Py_True : Py_False; + Py_INCREF(b); + PUSH(b); + PREDICT(POP_JUMP_IF_FALSE); + PREDICT(POP_JUMP_IF_TRUE); + DISPATCH(); + } + +#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ + "BaseException is not allowed" + + case TARGET(JUMP_IF_NOT_EXC_MATCH): { + PyObject *right = POP(); + PyObject *left = POP(); + if (PyTuple_Check(right)) { + Py_ssize_t i, length; + length = PyTuple_GET_SIZE(right); + for (i = 0; i < length; i++) { + PyObject *exc = PyTuple_GET_ITEM(right, i); + if (!PyExceptionClass_Check(exc)) { + _PyErr_SetString(tstate, PyExc_TypeError, + CANNOT_CATCH_MSG); + Py_DECREF(left); + Py_DECREF(right); + goto error; + } + } + } + else { + if (!PyExceptionClass_Check(right)) { + _PyErr_SetString(tstate, PyExc_TypeError, + CANNOT_CATCH_MSG); + Py_DECREF(left); + Py_DECREF(right); + goto error; + } + } + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(left); + Py_DECREF(right); + if (res > 0) { + /* Exception matches -- Do nothing */; + } + else if (res == 0) { + JUMPTO(oparg); + } + else { + goto error; + } + DISPATCH(); + } + + case TARGET(IMPORT_NAME): { + PyObject *name = GETITEM(names, oparg); + PyObject *fromlist = POP(); + PyObject *level = TOP(); + PyObject *res; + res = import_name(tstate, f, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(IMPORT_STAR): { + PyObject *from = POP(), *locals; + int err; + if (PyFrame_FastToLocalsWithError(f) < 0) { + Py_DECREF(from); + goto error; + } + + locals = f->f_locals; + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found during 'import *'"); + Py_DECREF(from); + goto error; + } + err = import_all_from(tstate, locals, from); + PyFrame_LocalsToFast(f, 0); + Py_DECREF(from); + if (err != 0) + goto error; + DISPATCH(); + } + + case TARGET(IMPORT_FROM): { + PyObject *name = GETITEM(names, oparg); + PyObject *from = TOP(); + PyObject *res; + res = import_from(tstate, from, name); + PUSH(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + case TARGET(JUMP_FORWARD): { + JUMPBY(oparg); + DISPATCH(); + } + + case TARGET(POP_JUMP_IF_FALSE): { + PREDICTED(POP_JUMP_IF_FALSE); + PyObject *cond = POP(); + int err; + if (Py_IsTrue(cond)) { + Py_DECREF(cond); + DISPATCH(); + } + if (Py_IsFalse(cond)) { + Py_DECREF(cond); + JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) + ; + else if (err == 0) { + JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + } + else + goto error; + DISPATCH(); + } + + case TARGET(POP_JUMP_IF_TRUE): { + PREDICTED(POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + int err; + if (Py_IsFalse(cond)) { + Py_DECREF(cond); + DISPATCH(); + } + if (Py_IsTrue(cond)) { + Py_DECREF(cond); + JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) { + JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + } + else if (err == 0) + ; + else + goto error; + DISPATCH(); + } + + case TARGET(JUMP_IF_FALSE_OR_POP): { + PyObject *cond = TOP(); + int err; + if (Py_IsTrue(cond)) { + STACK_SHRINK(1); + Py_DECREF(cond); + DISPATCH(); + } + if (Py_IsFalse(cond)) { + JUMPTO(oparg); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + if (err > 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + else if (err == 0) + JUMPTO(oparg); + else + goto error; + DISPATCH(); + } + + case TARGET(JUMP_IF_TRUE_OR_POP): { + PyObject *cond = TOP(); + int err; + if (Py_IsFalse(cond)) { + STACK_SHRINK(1); + Py_DECREF(cond); + DISPATCH(); + } + if (Py_IsTrue(cond)) { + JUMPTO(oparg); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + if (err > 0) { + JUMPTO(oparg); + } + else if (err == 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + else + goto error; + DISPATCH(); + } + + case TARGET(JUMP_ABSOLUTE): { + PREDICTED(JUMP_ABSOLUTE); + JUMPTO(oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(GET_LEN): { + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(TOP()); + if (len_i < 0) { + goto error; + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + goto error; + } + PUSH(len_o); + DISPATCH(); + } + + case TARGET(MATCH_CLASS): { + // Pop TOS. On success, set TOS to True and TOS1 to a tuple of + // attributes. On failure, set TOS to False. + PyObject *names = POP(); + PyObject *type = TOP(); + PyObject *subject = SECOND(); + assert(PyTuple_CheckExact(names)); + PyObject *attrs = match_class(tstate, subject, type, oparg, names); + Py_DECREF(names); + if (attrs) { + // Success! + assert(PyTuple_CheckExact(attrs)); + Py_DECREF(subject); + SET_SECOND(attrs); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + Py_DECREF(type); + SET_TOP(PyBool_FromLong(!!attrs)); + DISPATCH(); + } + + case TARGET(MATCH_MAPPING): { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + DISPATCH(); + } + + case TARGET(MATCH_SEQUENCE): { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + DISPATCH(); + } + + case TARGET(MATCH_KEYS): { + // On successful match for all keys, PUSH(values) and PUSH(True). + // Otherwise, PUSH(None) and PUSH(False). + PyObject *keys = TOP(); + PyObject *subject = SECOND(); + PyObject *values_or_none = match_keys(tstate, subject, keys); + if (values_or_none == NULL) { + goto error; + } + PUSH(values_or_none); + if (Py_IsNone(values_or_none)) { + Py_INCREF(Py_False); + PUSH(Py_False); + DISPATCH(); + } + assert(PyTuple_CheckExact(values_or_none)); + Py_INCREF(Py_True); + PUSH(Py_True); + DISPATCH(); + } + + case TARGET(COPY_DICT_WITHOUT_KEYS): { + // rest = dict(TOS1) + // for key in TOS: + // del rest[key] + // SET_TOP(rest) + PyObject *keys = TOP(); + PyObject *subject = SECOND(); + PyObject *rest = PyDict_New(); + if (rest == NULL || PyDict_Update(rest, subject)) { + Py_XDECREF(rest); + goto error; + } + // This may seem a bit inefficient, but keys is rarely big enough to + // actually impact runtime. + assert(PyTuple_CheckExact(keys)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(keys); i++) { + if (PyDict_DelItem(rest, PyTuple_GET_ITEM(keys, i))) { + Py_DECREF(rest); + goto error; + } + } + Py_DECREF(keys); + SET_TOP(rest); + DISPATCH(); + } + + case TARGET(GET_ITER): { + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable = TOP(); + PyObject *iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + SET_TOP(iter); + if (iter == NULL) + goto error; + PREDICT(FOR_ITER); + PREDICT(CALL_FUNCTION); + DISPATCH(); + } + + case TARGET(GET_YIELD_FROM_ITER): { + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable = TOP(); + PyObject *iter; + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + Py_DECREF(iterable); + SET_TOP(NULL); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + goto error; + } + } + else if (!PyGen_CheckExact(iterable)) { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + SET_TOP(iter); + if (iter == NULL) + goto error; + } + PREDICT(LOAD_CONST); + DISPATCH(); + } + + case TARGET(FOR_ITER): { + PREDICTED(FOR_ITER); + /* before: [iter]; after: [iter, iter()] *or* [] */ + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + PREDICT(STORE_FAST); + PREDICT(UNPACK_SEQUENCE); + DISPATCH(); + } + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + else if (tstate->c_tracefunc != NULL) { + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info); + } + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + STACK_SHRINK(1); + Py_DECREF(iter); + JUMPBY(oparg); + DISPATCH(); + } + + case TARGET(SETUP_FINALLY): { + PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg, + STACK_LEVEL()); + DISPATCH(); + } + + case TARGET(BEFORE_ASYNC_WITH): { + _Py_IDENTIFIER(__aenter__); + _Py_IDENTIFIER(__aexit__); + PyObject *mgr = TOP(); + PyObject *enter = special_lookup(tstate, mgr, &PyId___aenter__); + PyObject *res; + if (enter == NULL) { + goto error; + } + PyObject *exit = special_lookup(tstate, mgr, &PyId___aexit__); + if (exit == NULL) { + Py_DECREF(enter); + goto error; + } + SET_TOP(exit); + Py_DECREF(mgr); + res = _PyObject_CallNoArg(enter); + Py_DECREF(enter); + if (res == NULL) + goto error; + PUSH(res); + PREDICT(GET_AWAITABLE); + DISPATCH(); + } + + case TARGET(SETUP_ASYNC_WITH): { + PyObject *res = POP(); + /* Setup the finally block before pushing the result + of __aenter__ on the stack. */ + PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg, + STACK_LEVEL()); + PUSH(res); + DISPATCH(); + } + + case TARGET(SETUP_WITH): { + _Py_IDENTIFIER(__enter__); + _Py_IDENTIFIER(__exit__); + PyObject *mgr = TOP(); + PyObject *enter = special_lookup(tstate, mgr, &PyId___enter__); + PyObject *res; + if (enter == NULL) { + goto error; + } + PyObject *exit = special_lookup(tstate, mgr, &PyId___exit__); + if (exit == NULL) { + Py_DECREF(enter); + goto error; + } + SET_TOP(exit); + Py_DECREF(mgr); + res = _PyObject_CallNoArg(enter); + Py_DECREF(enter); + if (res == NULL) + goto error; + /* Setup the finally block before pushing the result + of __enter__ on the stack. */ + PyFrame_BlockSetup(f, SETUP_FINALLY, INSTR_OFFSET() + oparg, + STACK_LEVEL()); + + PUSH(res); + DISPATCH(); + } + + case TARGET(WITH_EXCEPT_START): { + /* At the top of the stack are 7 values: + - (TOP, SECOND, THIRD) = exc_info() + - (FOURTH, FIFTH, SIXTH) = previous exception for EXCEPT_HANDLER + - SEVENTH: the context.__exit__ bound method + We call SEVENTH(TOP, SECOND, THIRD). + Then we push again the TOP exception and the __exit__ + return value. + */ + PyObject *exit_func; + PyObject *exc, *val, *tb, *res; + + exc = TOP(); + val = SECOND(); + tb = THIRD(); + assert(!Py_IsNone(exc)); + assert(!PyLong_Check(exc)); + exit_func = PEEK(7); + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) + goto error; + + PUSH(res); + DISPATCH(); + } + + case TARGET(LOAD_METHOD): { + /* Designed to work in tandem with CALL_METHOD. */ + PyObject *name = GETITEM(names, oparg); + PyObject *obj = TOP(); + PyObject *meth = NULL; + + int meth_found = _PyObject_GetMethod(obj, name, &meth); + + if (meth == NULL) { + /* Most likely attribute wasn't found. */ + goto error; + } + + if (meth_found) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + + meth | self | arg1 | ... | argN + */ + SET_TOP(meth); + PUSH(obj); // self + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL_METHOD that it's not a method call. + + NULL | meth | arg1 | ... | argN + */ + SET_TOP(NULL); + Py_DECREF(obj); + PUSH(meth); + } + DISPATCH(); + } + + case TARGET(CALL_METHOD): { + /* Designed to work in tamdem with LOAD_METHOD. */ + PyObject **sp, *res, *meth; + + sp = stack_pointer; + + meth = PEEK(oparg + 2); + if (meth == NULL) { + /* `meth` is NULL when LOAD_METHOD thinks that it's not + a method call. + + Stack layout: + + ... | NULL | callable | arg1 | ... | argN + ^- TOP() + ^- (-oparg) + ^- (-oparg-1) + ^- (-oparg-2) + + `callable` will be POPed by call_function. + NULL will will be POPed manually later. + */ + res = call_function(tstate, &trace_info, &sp, oparg, NULL); + stack_pointer = sp; + (void)POP(); /* POP the NULL. */ + } + else { + /* This is a method call. Stack layout: + + ... | method | self | arg1 | ... | argN + ^- TOP() + ^- (-oparg) + ^- (-oparg-1) + ^- (-oparg-2) + + `self` and `method` will be POPed by call_function. + We'll be passing `oparg + 1` to call_function, to + make it accept the `self` as a first argument. + */ + res = call_function(tstate, &trace_info, &sp, oparg + 1, NULL); + stack_pointer = sp; + } + + PUSH(res); + if (res == NULL) + goto error; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(CALL_FUNCTION): { + PREDICTED(CALL_FUNCTION); + PyObject **sp, *res; + sp = stack_pointer; + res = call_function(tstate, &trace_info, &sp, oparg, NULL); + stack_pointer = sp; + PUSH(res); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(CALL_FUNCTION_KW): { + PyObject **sp, *res, *names; + + names = POP(); + assert(PyTuple_Check(names)); + assert(PyTuple_GET_SIZE(names) <= oparg); + /* We assume without checking that names contains only strings */ + sp = stack_pointer; + res = call_function(tstate, &trace_info, &sp, oparg, names); + stack_pointer = sp; + PUSH(res); + Py_DECREF(names); + + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(CALL_FUNCTION_EX): { + PREDICTED(CALL_FUNCTION_EX); + PyObject *func, *callargs, *kwargs = NULL, *result; + if (oparg & 0x01) { + kwargs = POP(); + if (!PyDict_CheckExact(kwargs)) { + PyObject *d = PyDict_New(); + if (d == NULL) + goto error; + if (_PyDict_MergeEx(d, kwargs, 2) < 0) { + Py_DECREF(d); + format_kwargs_error(tstate, SECOND(), kwargs); + Py_DECREF(kwargs); + goto error; + } + Py_DECREF(kwargs); + kwargs = d; + } + assert(PyDict_CheckExact(kwargs)); + } + callargs = POP(); + func = TOP(); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + Py_DECREF(callargs); + goto error; + } + Py_SETREF(callargs, PySequence_Tuple(callargs)); + if (callargs == NULL) { + goto error; + } + } + assert(PyTuple_CheckExact(callargs)); + + result = do_call_core(tstate, &trace_info, func, callargs, kwargs); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + + SET_TOP(result); + if (result == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + case TARGET(MAKE_FUNCTION): { + PyObject *qualname = POP(); + PyObject *codeobj = POP(); + PyFunctionObject *func = (PyFunctionObject *) + PyFunction_NewWithQualName(codeobj, f->f_globals, qualname); + + Py_DECREF(codeobj); + Py_DECREF(qualname); + if (func == NULL) { + goto error; + } + + if (oparg & 0x08) { + assert(PyTuple_CheckExact(TOP())); + func->func_closure = POP(); + } + if (oparg & 0x04) { + assert(PyTuple_CheckExact(TOP())); + func->func_annotations = POP(); + } + if (oparg & 0x02) { + assert(PyDict_CheckExact(TOP())); + func->func_kwdefaults = POP(); + } + if (oparg & 0x01) { + assert(PyTuple_CheckExact(TOP())); + func->func_defaults = POP(); + } + + PUSH((PyObject *)func); + DISPATCH(); + } + + case TARGET(BUILD_SLICE): { + PyObject *start, *stop, *step, *slice; + if (oparg == 3) + step = POP(); + else + step = NULL; + stop = POP(); + start = TOP(); + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + SET_TOP(slice); + if (slice == NULL) + goto error; + DISPATCH(); + } + + case TARGET(FORMAT_VALUE): { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = POP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_NONE: conv_fn = NULL; break; + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + default: + _PyErr_Format(tstate, PyExc_SystemError, + "unexpected conversion flag %d", + which_conversion); + goto error; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn != NULL) { + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (result == NULL) { + goto error; + } + } + + PUSH(result); + DISPATCH(); + } + + case TARGET(ROT_N): { + PyObject *top = TOP(); + memmove(&PEEK(oparg - 1), &PEEK(oparg), + sizeof(PyObject*) * (oparg - 1)); + PEEK(oparg) = top; + DISPATCH(); + } + + case TARGET(EXTENDED_ARG): { + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; + goto dispatch_opcode; + } + + +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#endif + default: + fprintf(stderr, + "XXX lineno: %d, opcode: %d\n", + PyFrame_GetLineNumber(f), + opcode); + _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode"); + goto error; + + } /* switch */ + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + PyTraceBack_Here(f); + + if (tstate->c_tracefunc != NULL) { + /* Make sure state is set to FRAME_EXECUTING for tracing */ + assert(f->f_state == FRAME_EXECUTING); + f->f_state = FRAME_UNWINDING; + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, + tstate, f, &trace_info); + } +exception_unwind: + f->f_state = FRAME_UNWINDING; + /* Unwind stacks if an exception occurred */ + while (f->f_iblock > 0) { + /* Pop the current block. */ + PyTryBlock *b = &f->f_blockstack[--f->f_iblock]; + + if (b->b_type == EXCEPT_HANDLER) { + UNWIND_EXCEPT_HANDLER(b); + continue; + } + UNWIND_BLOCK(b); + if (b->b_type == SETUP_FINALLY) { + PyObject *exc, *val, *tb; + int handler = b->b_handler; + _PyErr_StackItem *exc_info = tstate->exc_info; + /* Beware, this invalidates all b->b_* fields */ + PyFrame_BlockSetup(f, EXCEPT_HANDLER, f->f_lasti, STACK_LEVEL()); + PUSH(exc_info->exc_traceback); + PUSH(exc_info->exc_value); + if (exc_info->exc_type != NULL) { + PUSH(exc_info->exc_type); + } + else { + Py_INCREF(Py_None); + PUSH(Py_None); + } + _PyErr_Fetch(tstate, &exc, &val, &tb); + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + _PyErr_NormalizeException(tstate, &exc, &val, &tb); + if (tb != NULL) + PyException_SetTraceback(val, tb); + else + PyException_SetTraceback(val, Py_None); + Py_INCREF(exc); + exc_info->exc_type = exc; + Py_INCREF(val); + exc_info->exc_value = val; + exc_info->exc_traceback = tb; + if (tb == NULL) + tb = Py_None; + Py_INCREF(tb); + PUSH(tb); + PUSH(val); + PUSH(exc); + JUMPTO(handler); + /* Resume normal execution */ + f->f_state = FRAME_EXECUTING; + goto main_loop; + } + } /* unwind stack */ + + /* End the loop as we still have an error */ + break; + } /* main loop */ + + assert(retval == NULL); + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + while (!EMPTY()) { + PyObject *o = POP(); + Py_XDECREF(o); + } + f->f_stackdepth = 0; + f->f_state = FRAME_RAISED; +exiting: + if (trace_info.cframe.use_tracing) { + if (tstate->c_tracefunc) { + if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, + tstate, f, &trace_info, PyTrace_RETURN, retval)) { + Py_CLEAR(retval); + } + } + if (tstate->c_profilefunc) { + if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, + tstate, f, &trace_info, PyTrace_RETURN, retval)) { + Py_CLEAR(retval); + } + } + } + + /* pop frame */ +exit_eval_frame: + /* Restore previous cframe */ + tstate->cframe = trace_info.cframe.previous; + tstate->cframe->use_tracing = trace_info.cframe.use_tracing; + + if (PyDTrace_FUNCTION_RETURN_ENABLED()) + dtrace_function_return(f); + _Py_LeaveRecursiveCall(tstate); + tstate->frame = f->f_back; + + return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); +} + +static PyObject * +make_coro(PyFrameConstructor *con, PyFrameObject *f) +{ + assert (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)); + PyObject *gen; + int is_coro = ((PyCodeObject *)con->fc_code)->co_flags & CO_COROUTINE; + + /* Don't need to keep the reference to f_back, it will be set + * when the generator is resumed. */ + Py_CLEAR(f->f_back); + + /* Create a new generator that owns the ready to run frame + * and return that as the value. */ + if (is_coro) { + gen = PyCoro_New(f, con->fc_name, con->fc_qualname); + } else if (((PyCodeObject *)con->fc_code)->co_flags & CO_ASYNC_GENERATOR) { + gen = PyAsyncGen_New(f, con->fc_name, con->fc_qualname); + } else { + gen = PyGen_NewWithQualName(f, con->fc_name, con->fc_qualname); + } + if (gen == NULL) { + return NULL; + } + + _PyObject_GC_TRACK(f); + + return gen; +} + +PyObject * +_PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames) +{ + PyFrameObject *f = _PyEval_MakeFrameVector( + tstate, con, locals, args, argcount, kwnames); + if (f == NULL) { + return NULL; + } + if (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { + return make_coro(con, f); + } + PyObject *retval = _PyEval_EvalFrame(tstate, f, 0); + + /* decref'ing the frame can cause __del__ methods to get invoked, + which can call back into Python. While we're done with the + current Python frame (f), the associated C stack is still in use, + so recursion_depth must be boosted for the duration. + */ + if (Py_REFCNT(f) > 1) { + Py_DECREF(f); + _PyObject_GC_TRACK(f); + } + else { + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + } + return retval; +} diff --git a/lecture20/cpython/3.10.8/opcode.h b/lecture20/cpython/3.10.8/opcode.h @@ -0,0 +1,172 @@ +/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */ +#ifndef Py_OPCODE_H +#define Py_OPCODE_H +#ifdef __cplusplus +extern "C" { +#endif + + + /* Instruction opcodes for compiled code */ +#define POP_TOP 1 +#define ROT_TWO 2 +#define ROT_THREE 3 +#define DUP_TOP 4 +#define DUP_TOP_TWO 5 +#define ROT_FOUR 6 +#define NOP 9 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define BINARY_MATRIX_MULTIPLY 16 +#define INPLACE_MATRIX_MULTIPLY 17 +#define BINARY_POWER 19 +#define BINARY_MULTIPLY 20 +#define BINARY_MODULO 22 +#define BINARY_ADD 23 +#define BINARY_SUBTRACT 24 +#define BINARY_SUBSCR 25 +#define BINARY_FLOOR_DIVIDE 26 +#define BINARY_TRUE_DIVIDE 27 +#define INPLACE_FLOOR_DIVIDE 28 +#define INPLACE_TRUE_DIVIDE 29 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define COPY_DICT_WITHOUT_KEYS 34 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define END_ASYNC_FOR 54 +#define INPLACE_ADD 55 +#define INPLACE_SUBTRACT 56 +#define INPLACE_MULTIPLY 57 +#define INPLACE_MODULO 59 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define BINARY_LSHIFT 62 +#define BINARY_RSHIFT 63 +#define BINARY_AND 64 +#define BINARY_XOR 65 +#define BINARY_OR 66 +#define INPLACE_POWER 67 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define YIELD_FROM 72 +#define GET_AWAITABLE 73 +#define LOAD_ASSERTION_ERROR 74 +#define INPLACE_LSHIFT 75 +#define INPLACE_RSHIFT 76 +#define INPLACE_AND 77 +#define INPLACE_XOR 78 +#define INPLACE_OR 79 +#define LIST_TO_TUPLE 82 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 +#define SETUP_ANNOTATIONS 85 +#define YIELD_VALUE 86 +#define POP_BLOCK 87 +#define POP_EXCEPT 89 +#define HAVE_ARGUMENT 90 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define ROT_N 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define JUMP_IF_FALSE_OR_POP 111 +#define JUMP_IF_TRUE_OR_POP 112 +#define JUMP_ABSOLUTE 113 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define JUMP_IF_NOT_EXC_MATCH 121 +#define SETUP_FINALLY 122 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define GEN_START 129 +#define RAISE_VARARGS 130 +#define CALL_FUNCTION 131 +#define MAKE_FUNCTION 132 +#define BUILD_SLICE 133 +#define LOAD_CLOSURE 135 +#define LOAD_DEREF 136 +#define STORE_DEREF 137 +#define DELETE_DEREF 138 +#define CALL_FUNCTION_KW 141 +#define CALL_FUNCTION_EX 142 +#define SETUP_WITH 143 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define LOAD_CLASSDEREF 148 +#define MATCH_CLASS 152 +#define SETUP_ASYNC_WITH 154 +#define FORMAT_VALUE 155 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define LOAD_METHOD 160 +#define CALL_METHOD 161 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#ifdef NEED_OPCODE_JUMP_TABLES +static uint32_t _PyOpcode_RelativeJump[8] = { + 0U, + 0U, + 536870912U, + 67125248U, + 67141632U, + 0U, + 0U, + 0U, +}; +static uint32_t _PyOpcode_Jump[8] = { + 0U, + 0U, + 536870912U, + 101695488U, + 67141632U, + 0U, + 0U, + 0U, +}; +#endif /* OPCODE_TABLES */ + +/* EXCEPT_HANDLER is a special, implicit block type which is created when + entering an except handler. It is not an opcode but we define it here + as we want it to be available to both frameobject.c and ceval.c, while + remaining private.*/ +#define EXCEPT_HANDLER 257 + +#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_H */ diff --git a/lecture20/cpython/3.11.0rc2/ceval.c b/lecture20/cpython/3.11.0rc2/ceval.c @@ -0,0 +1,4232 @@ +// XXX: [fabianw@seas.harvard.edu; 2022-11-08] This code has been shortened to +// fit the context of the lecture. The full code for this file can be found +// here: +// +// https://github.com/python/cpython/blob/v3.11.0rc2/Python/ceval.c + +PyObject* _Py_HOT_FUNCTION +_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + CALL_STAT_INC(pyeval_calls); + +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" +#endif + +#ifdef Py_STATS + int lastopcode = 0; +#endif + // opcode is an 8-bit value to improve the code generated by MSVC + // for the big switch below (in combination with the EXTRA_CASES macro). + uint8_t opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ + _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; +#ifdef LLTRACE + int lltrace = 0; +#endif + + _PyCFrame cframe; + CallShape call_shape; + call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions. + + /* WARNING: Because the _PyCFrame lives on the C stack, + * but can be accessed from a heap allocated object (tstate) + * strict stack discipline must be maintained. + */ + _PyCFrame *prev_cframe = tstate->cframe; + cframe.use_tracing = prev_cframe->use_tracing; + cframe.previous = prev_cframe; + tstate->cframe = &cframe; + + frame->is_entry = true; + /* Push frame */ + frame->previous = prev_cframe->current_frame; + cframe.current_frame = frame; + + /* support for generator.throw() */ + if (throwflag) { + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + tstate->recursion_remaining--; + goto exit_unwind; + } + TRACE_FUNCTION_THROW_ENTRY(); + DTRACE_FUNCTION_ENTRY(); + goto resume_with_error; + } + + /* Local "register" variables. + * These are cached values from the frame and code object. */ + + PyObject *names; + PyObject *consts; + _Py_CODEUNIT *first_instr; + _Py_CODEUNIT *next_instr; + PyObject **stack_pointer; + +/* Sets the above local variables from the frame */ +#define SET_LOCALS_FROM_FRAME() \ + { \ + PyCodeObject *co = frame->f_code; \ + names = co->co_names; \ + consts = co->co_consts; \ + first_instr = _PyCode_CODE(co); \ + } \ + assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ + /* Jump back to the last instruction executed... */ \ + next_instr = frame->prev_instr + 1; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + /* Set stackdepth to -1. \ + Update when returning or calling trace function. \ + Having stackdepth <= 0 ensures that invalid \ + values are not visible to the cycle GC. \ + We choose -1 rather than 0 to assist debugging. \ + */ \ + frame->stacktop = -1; + + +start_frame: + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + tstate->recursion_remaining--; + goto exit_unwind; + } + +resume_frame: + SET_LOCALS_FROM_FRAME(); + +#ifdef LLTRACE + { + int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); + if (r < 0) { + goto exit_unwind; + } + lltrace = r; + } + if (lltrace) { + lltrace_resume_frame(frame); + } +#endif + +#ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif + + DISPATCH(); + +handle_eval_breaker: + + /* Do periodic things, like check for signals and async I/0. + * We need to do reasonably frequently, but not too frequently. + * All loops should include a check of the eval breaker. + * We also check on return from any builtin function. + */ + if (eval_frame_handle_pending(tstate) != 0) { + goto error; + } + DISPATCH(); + + { + /* Start instructions */ +#if USE_COMPUTED_GOTOS + { +#else + dispatch_opcode: + switch (opcode) { +#endif + + /* BEWARE! + It is essential that any operation that fails must goto error + and that all operation that succeed call DISPATCH() ! */ + + TARGET(NOP) { + DISPATCH(); + } + + TARGET(RESUME) { + _PyCode_Warmup(frame->f_code); + JUMP_TO_INSTRUCTION(RESUME_QUICK); + } + + TARGET(RESUME_QUICK) { + PREDICTED(RESUME_QUICK); + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + DISPATCH(); + } + + TARGET(LOAD_CLOSURE) { + /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_FAST) { + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_CONST) { + PREDICTED(LOAD_CONST); + PyObject *value = GETITEM(consts, oparg); + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(STORE_FAST) { + PREDICTED(STORE_FAST); + PyObject *value = POP(); + SETLOCAL(oparg, value); + DISPATCH(); + } + + TARGET(LOAD_FAST__LOAD_FAST) { + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + NEXTOPARG(); + next_instr++; + Py_INCREF(value); + PUSH(value); + value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_FAST__LOAD_CONST) { + PyObject *value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + NEXTOPARG(); + next_instr++; + Py_INCREF(value); + PUSH(value); + value = GETITEM(consts, oparg); + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(STORE_FAST__LOAD_FAST) { + PyObject *value = POP(); + SETLOCAL(oparg, value); + NEXTOPARG(); + next_instr++; + value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(STORE_FAST__STORE_FAST) { + PyObject *value = POP(); + SETLOCAL(oparg, value); + NEXTOPARG(); + next_instr++; + value = POP(); + SETLOCAL(oparg, value); + DISPATCH(); + } + + TARGET(LOAD_CONST__LOAD_FAST) { + PyObject *value = GETITEM(consts, oparg); + NEXTOPARG(); + next_instr++; + Py_INCREF(value); + PUSH(value); + value = GETLOCAL(oparg); + if (value == NULL) { + goto unbound_local_error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(POP_TOP) { + PyObject *value = POP(); + Py_DECREF(value); + DISPATCH(); + } + + TARGET(PUSH_NULL) { + /* Use BASIC_PUSH as NULL is not a valid object pointer */ + BASIC_PUSH(NULL); + DISPATCH(); + } + + TARGET(UNARY_POSITIVE) { + PyObject *value = TOP(); + PyObject *res = PyNumber_Positive(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + PyObject *value = TOP(); + PyObject *res = PyNumber_Negative(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + TARGET(UNARY_NOT) { + PyObject *value = TOP(); + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err == 0) { + Py_INCREF(Py_True); + SET_TOP(Py_True); + DISPATCH(); + } + else if (err > 0) { + Py_INCREF(Py_False); + SET_TOP(Py_False); + DISPATCH(); + } + STACK_SHRINK(1); + goto error; + } + + TARGET(UNARY_INVERT) { + PyObject *value = TOP(); + PyObject *res = PyNumber_Invert(value); + Py_DECREF(value); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(prod); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dprod = ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + PyObject *prod = PyFloat_FromDouble(dprod); + SET_SECOND(prod); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + STACK_SHRINK(1); + if (prod == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(sub); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + STACK_SHRINK(1); + if (sub == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; + PyObject *sub = PyFloat_FromDouble(dsub); + SET_SECOND(sub); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + STACK_SHRINK(1); + if (sub == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_UNICODE) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + PyObject *res = PyUnicode_Concat(left, right); + STACK_SHRINK(1); + SET_TOP(res); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (TOP() == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; + assert(_Py_OPCODE(true_next) == STORE_FAST || + _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST); + PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next)); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + STACK_SHRINK(2); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) { + goto error; + } + // The STORE_FAST is already done. + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_FLOAT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + double dsum = ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + PyObject *sum = PyFloat_FromDouble(dsum); + SET_SECOND(sum); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + STACK_SHRINK(1); + if (sum == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_INT) { + assert(cframe.use_tracing == 0); + PyObject *left = SECOND(); + PyObject *right = TOP(); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); + STAT_INC(BINARY_OP, hit); + PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + SET_SECOND(sum); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + STACK_SHRINK(1); + if (sum == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_SUBSCR) { + PREDICTED(BINARY_SUBSCR); + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_ADAPTIVE) { + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(BINARY_SUBSCR); + } + } + + TARGET(BINARY_SUBSCR_LIST_INT) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *list = SECOND(); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + + // Deopt unless 0 <= sub < PyList_Size(list) + Py_ssize_t signed_magnitude = Py_SIZE(sub); + DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); + Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + STACK_SHRINK(1); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + SET_TOP(res); + Py_DECREF(list); + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_TUPLE_INT) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *tuple = SECOND(); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + + // Deopt unless 0 <= sub < PyTuple_Size(list) + Py_ssize_t signed_magnitude = Py_SIZE(sub); + DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR); + assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0); + Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + STACK_SHRINK(1); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + SET_TOP(res); + Py_DECREF(tuple); + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_DICT) { + assert(cframe.use_tracing == 0); + PyObject *dict = SECOND(); + DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + PyObject *sub = TOP(); + PyObject *res = PyDict_GetItemWithError(dict, sub); + if (res == NULL) { + goto binary_subscr_dict_error; + } + Py_INCREF(res); + STACK_SHRINK(1); + Py_DECREF(sub); + SET_TOP(res); + Py_DECREF(dict); + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_GETITEM) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + PyObject *cached = ((PyHeapTypeObject *)tp)->_spec_cache.getitem; + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + DEOPT_IF(getitem->func_version != cache->func_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE; + assert(code->co_argcount == 2); + _PyInterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size); + if (new_frame == NULL) { + goto error; + } + CALL_STAT_INC(frames_pushed); + Py_INCREF(getitem); + _PyFrame_InitializeSpecials(new_frame, getitem, + NULL, code->co_nlocalsplus); + STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + for (int i = 2; i < code->co_nlocalsplus; i++) { + new_frame->localsplus[i] = NULL; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + frame = cframe.current_frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + TARGET(LIST_APPEND) { + PyObject *v = POP(); + PyObject *list = PEEK(oparg); + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) + goto error; + PREDICT(JUMP_BACKWARD_QUICK); + DISPATCH(); + } + + TARGET(SET_ADD) { + PyObject *v = POP(); + PyObject *set = PEEK(oparg); + int err; + err = PySet_Add(set, v); + Py_DECREF(v); + if (err != 0) + goto error; + PREDICT(JUMP_BACKWARD_QUICK); + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + PREDICTED(STORE_SUBSCR); + PyObject *sub = TOP(); + PyObject *container = SECOND(); + PyObject *v = THIRD(); + int err; + STACK_SHRINK(3); + /* container[sub] = v */ + err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DISPATCH(); + } + + TARGET(STORE_SUBSCR_ADAPTIVE) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(STORE_SUBSCR); + } + } + + TARGET(STORE_SUBSCR_LIST_INT) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *list = SECOND(); + PyObject *value = THIRD(); + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + STACK_SHRINK(3); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DISPATCH(); + } + + TARGET(STORE_SUBSCR_DICT) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *dict = SECOND(); + PyObject *value = THIRD(); + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STACK_SHRINK(3); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + int err; + STACK_SHRINK(2); + /* del container[sub] */ + err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err != 0) + goto error; + DISPATCH(); + } + + TARGET(PRINT_EXPR) { + PyObject *value = POP(); + PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook)); + PyObject *res; + if (hook == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "lost sys.displayhook"); + Py_DECREF(value); + goto error; + } + res = PyObject_CallOneArg(hook, value); + Py_DECREF(value); + if (res == NULL) + goto error; + Py_DECREF(res); + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = POP(); /* cause */ + /* fall through */ + case 1: + exc = POP(); /* exc */ + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; + } + goto error; + } + + TARGET(RETURN_VALUE) { + PyObject *retval = POP(); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_EXIT(); + DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallTstate(tstate); + if (!frame->is_entry) { + frame = cframe.current_frame = pop_frame(tstate, frame); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + /* Restore previous cframe and return. */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; + assert(tstate->cframe->current_frame == frame->previous); + assert(!_PyErr_Occurred(tstate)); + return retval; + } + + TARGET(GET_AITER) { + unaryfunc getter = NULL; + PyObject *iter = NULL; + PyObject *obj = TOP(); + PyTypeObject *type = Py_TYPE(obj); + + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + + if (getter != NULL) { + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) { + SET_TOP(NULL); + goto error; + } + } + else { + SET_TOP(NULL); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + goto error; + } + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + + SET_TOP(NULL); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + goto error; + } + + SET_TOP(iter); + DISPATCH(); + } + + TARGET(GET_ANEXT) { + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyObject *awaitable = NULL; + PyObject *aiter = TOP(); + PyTypeObject *type = Py_TYPE(aiter); + + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + goto error; + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + goto error; + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + goto error; + } + + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + + Py_DECREF(next_iter); + goto error; + } else { + Py_DECREF(next_iter); + } + } + + PUSH(awaitable); + PREDICT(LOAD_CONST); + DISPATCH(); + } + + TARGET(GET_AWAITABLE) { + PREDICTED(GET_AWAITABLE); + PyObject *iterable = TOP(); + PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + + if (iter == NULL) { + format_awaitable_error(tstate, Py_TYPE(iterable), oparg); + } + + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } + } + + SET_TOP(iter); /* Even if it's NULL */ + + if (iter == NULL) { + goto error; + } + + PREDICT(LOAD_CONST); + DISPATCH(); + } + + TARGET(SEND) { + assert(frame->is_entry); + assert(STACK_LEVEL() >= 2); + PyObject *v = POP(); + PyObject *receiver = TOP(); + PySendResult gen_status; + PyObject *retval; + if (tstate->c_tracefunc == NULL) { + gen_status = PyIter_Send(receiver, v, &retval); + } else { + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (tstate->c_tracefunc != NULL + && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + gen_status = PYGEN_RETURN; + } + else { + gen_status = PYGEN_ERROR; + } + } + else { + gen_status = PYGEN_NEXT; + } + } + Py_DECREF(v); + if (gen_status == PYGEN_ERROR) { + assert(retval == NULL); + goto error; + } + if (gen_status == PYGEN_RETURN) { + assert(retval != NULL); + Py_DECREF(receiver); + SET_TOP(retval); + JUMPBY(oparg); + DISPATCH(); + } + assert(gen_status == PYGEN_NEXT); + assert(retval != NULL); + PUSH(retval); + DISPATCH(); + } + + TARGET(ASYNC_GEN_WRAP) { + PyObject *v = TOP(); + assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); + PyObject *w = _PyAsyncGenValueWrapperNew(v); + if (w == NULL) { + goto error; + } + SET_TOP(w); + Py_DECREF(v); + DISPATCH(); + } + + TARGET(YIELD_VALUE) { + assert(frame->is_entry); + PyObject *retval = POP(); + _PyFrame_GetGenerator(frame)->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_EXIT(); + DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallTstate(tstate); + /* Restore previous cframe and return. */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; + assert(tstate->cframe->current_frame == frame->previous); + assert(!_PyErr_Occurred(tstate)); + return retval; + } + + TARGET(POP_EXCEPT) { + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *value = exc_info->exc_value; + exc_info->exc_value = POP(); + Py_XDECREF(value); + DISPATCH(); + } + + TARGET(RERAISE) { + if (oparg) { + PyObject *lasti = PEEK(oparg + 1); + if (PyLong_Check(lasti)) { + frame->prev_instr = first_instr + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + goto error; + } + } + PyObject *val = POP(); + assert(val && PyExceptionInstance_Check(val)); + PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val)); + PyObject *tb = PyException_GetTraceback(val); + _PyErr_Restore(tstate, exc, val, tb); + goto exception_unwind; + } + + TARGET(PREP_RERAISE_STAR) { + PyObject *excs = POP(); + assert(PyList_Check(excs)); + PyObject *orig = POP(); + + PyObject *val = _PyExc_PrepReraiseStar(orig, excs); + Py_DECREF(excs); + Py_DECREF(orig); + + if (val == NULL) { + goto error; + } + + PUSH(val); + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + PyObject *val = POP(); + assert(val && PyExceptionInstance_Check(val)); + if (PyErr_GivenExceptionMatches(val, PyExc_StopAsyncIteration)) { + Py_DECREF(val); + Py_DECREF(POP()); + DISPATCH(); + } + else { + PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val)); + PyObject *tb = PyException_GetTraceback(val); + _PyErr_Restore(tstate, exc, val, tb); + goto exception_unwind; + } + } + + TARGET(LOAD_ASSERTION_ERROR) { + PyObject *value = PyExc_AssertionError; + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + PyObject *bc; + if (PyDict_CheckExact(BUILTINS())) { + bc = _PyDict_GetItemWithError(BUILTINS(), + &_Py_ID(__build_class__)); + if (bc == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + } + goto error; + } + Py_INCREF(bc); + } + else { + bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__)); + if (bc == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + goto error; + } + } + PUSH(bc); + DISPATCH(); + } + + TARGET(STORE_NAME) { + PyObject *name = GETITEM(names, oparg); + PyObject *v = POP(); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + goto error; + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err != 0) + goto error; + DISPATCH(); + } + + TARGET(DELETE_NAME) { + PyObject *name = GETITEM(names, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + goto error; + } + err = PyObject_DelItem(ns, name); + if (err != 0) { + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + goto error; + } + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { + PREDICTED(UNPACK_SEQUENCE); + PyObject *seq = POP(); + PyObject **top = stack_pointer + oparg; + if (!unpack_iterable(tstate, seq, oparg, -1, top)) { + Py_DECREF(seq); + goto error; + } + STACK_GROW(oparg); + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *seq = TOP(); + next_instr--; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(UNPACK_SEQUENCE); + } + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + PyObject *seq = TOP(); + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + SET_TOP(Py_NewRef(PyTuple_GET_ITEM(seq, 1))); + PUSH(Py_NewRef(PyTuple_GET_ITEM(seq, 0))); + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + PyObject *seq = TOP(); + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + STACK_SHRINK(1); + PyObject **items = _PyTuple_ITEMS(seq); + while (oparg--) { + PUSH(Py_NewRef(items[oparg])); + } + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + PyObject *seq = TOP(); + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + STACK_SHRINK(1); + PyObject **items = _PyList_ITEMS(seq); + while (oparg--) { + PUSH(Py_NewRef(items[oparg])); + } + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); + DISPATCH(); + } + + TARGET(UNPACK_EX) { + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject *seq = POP(); + PyObject **top = stack_pointer + totalargs; + if (!unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top)) { + Py_DECREF(seq); + goto error; + } + STACK_GROW(totalargs); + Py_DECREF(seq); + DISPATCH(); + } + + TARGET(STORE_ATTR) { + PREDICTED(STORE_ATTR); + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + DISPATCH(); + } + + TARGET(DELETE_ATTR) { + PyObject *name = GETITEM(names, oparg); + PyObject *owner = POP(); + int err; + err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + Py_DECREF(owner); + if (err != 0) + goto error; + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + PyObject *name = GETITEM(names, oparg); + PyObject *v = POP(); + int err; + err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err != 0) + goto error; + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + PyObject *name = GETITEM(names, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + DISPATCH(); + } + + TARGET(LOAD_NAME) { + PyObject *name = GETITEM(names, oparg); + PyObject *locals = LOCALS(); + PyObject *v; + if (locals == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when loading %R", name); + goto error; + } + if (PyDict_CheckExact(locals)) { + v = PyDict_GetItemWithError(locals, name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + } + else { + v = PyObject_GetItem(locals, name); + if (v == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) + goto error; + _PyErr_Clear(tstate); + } + } + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + else { + if (PyDict_CheckExact(BUILTINS())) { + v = PyDict_GetItemWithError(BUILTINS(), name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + Py_INCREF(v); + } + else { + v = PyObject_GetItem(BUILTINS(), name); + if (v == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + } + } + } + PUSH(v); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + PREDICTED(LOAD_GLOBAL); + int push_null = oparg & 1; + PEEK(0) = NULL; + PyObject *name = GETITEM(names, oparg>>1); + PyObject *v; + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + format_exc_check_arg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + Py_INCREF(v); + } + else { + /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ + v = PyObject_GetItem(GLOBALS(), name); + if (v == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + + /* namespace 2: builtins */ + v = PyObject_GetItem(BUILTINS(), name); + if (v == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + format_exc_check_arg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + goto error; + } + } + } + /* Skip over inline cache */ + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + STACK_GROW(push_null); + PUSH(v); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(LOAD_GLOBAL); + } + } + + TARGET(LOAD_GLOBAL_MODULE) { + assert(cframe.use_tracing == 0); + DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + uint32_t version = read_u32(cache->module_keys_version); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyObject *res = entries[cache->index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + int push_null = oparg & 1; + PEEK(0) = NULL; + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + STAT_INC(LOAD_GLOBAL, hit); + STACK_GROW(push_null+1); + Py_INCREF(res); + SET_TOP(res); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + assert(cframe.use_tracing == 0); + DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); + PyDictObject *mdict = (PyDictObject *)GLOBALS(); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + uint32_t mod_version = read_u32(cache->module_keys_version); + uint16_t bltn_version = cache->builtin_keys_version; + DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); + DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(bdict->ma_keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + PyObject *res = entries[cache->index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + int push_null = oparg & 1; + PEEK(0) = NULL; + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_GLOBAL); + STAT_INC(LOAD_GLOBAL, hit); + STACK_GROW(push_null+1); + Py_INCREF(res); + SET_TOP(res); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + PyObject *v = GETLOCAL(oparg); + if (v != NULL) { + SETLOCAL(oparg, NULL); + DISPATCH(); + } + goto unbound_local_error; + } + + TARGET(MAKE_CELL) { + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + goto resume_with_error; + } + SETLOCAL(oparg, cell); + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + if (oldobj != NULL) { + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); + DISPATCH(); + } + format_exc_unbound(tstate, frame->f_code, oparg); + goto error; + } + + TARGET(LOAD_CLASSDEREF) { + PyObject *name, *value, *locals = LOCALS(); + assert(locals); + assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); + name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg); + if (PyDict_CheckExact(locals)) { + value = PyDict_GetItemWithError(locals, name); + if (value != NULL) { + Py_INCREF(value); + } + else if (_PyErr_Occurred(tstate)) { + goto error; + } + } + else { + value = PyObject_GetItem(locals, name); + if (value == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + } + } + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + format_exc_unbound(tstate, frame->f_code, oparg); + goto error; + } + Py_INCREF(value); + } + PUSH(value); + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + PyObject *cell = GETLOCAL(oparg); + PyObject *value = PyCell_GET(cell); + if (value == NULL) { + format_exc_unbound(tstate, frame->f_code, oparg); + goto error; + } + Py_INCREF(value); + PUSH(value); + DISPATCH(); + } + + TARGET(STORE_DEREF) { + PyObject *v = POP(); + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + /* Copy closure variables to free variables */ + PyCodeObject *co = frame->f_code; + PyObject *closure = frame->f_func->func_closure; + int offset = co->co_nlocals + co->co_nplaincellvars; + assert(oparg == co->co_nfreevars); + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + Py_INCREF(o); + frame->localsplus[offset + i] = o; + } + DISPATCH(); + } + + TARGET(BUILD_STRING) { + PyObject *str; + str = _PyUnicode_JoinArray(&_Py_STR(empty), + stack_pointer - oparg, oparg); + if (str == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + Py_DECREF(item); + } + PUSH(str); + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + PyObject *tup = PyTuple_New(oparg); + if (tup == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + PyTuple_SET_ITEM(tup, oparg, item); + } + PUSH(tup); + DISPATCH(); + } + + TARGET(BUILD_LIST) { + PyObject *list = PyList_New(oparg); + if (list == NULL) + goto error; + while (--oparg >= 0) { + PyObject *item = POP(); + PyList_SET_ITEM(list, oparg, item); + } + PUSH(list); + DISPATCH(); + } + + TARGET(LIST_TO_TUPLE) { + PyObject *list = POP(); + PyObject *tuple = PyList_AsTuple(list); + Py_DECREF(list); + if (tuple == NULL) { + goto error; + } + PUSH(tuple); + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + PyObject *iterable = POP(); + PyObject *list = PEEK(oparg); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + goto error; + } + Py_DECREF(none_val); + Py_DECREF(iterable); + DISPATCH(); + } + + TARGET(SET_UPDATE) { + PyObject *iterable = POP(); + PyObject *set = PEEK(oparg); + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) { + goto error; + } + DISPATCH(); + } + + TARGET(BUILD_SET) { + PyObject *set = PySet_New(NULL); + int err = 0; + int i; + if (set == NULL) + goto error; + for (i = oparg; i > 0; i--) { + PyObject *item = PEEK(i); + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + STACK_SHRINK(oparg); + if (err != 0) { + Py_DECREF(set); + goto error; + } + PUSH(set); + DISPATCH(); + } + + TARGET(BUILD_MAP) { + PyObject *map = _PyDict_FromItems( + &PEEK(2*oparg), 2, + &PEEK(2*oparg - 1), 2, + oparg); + if (map == NULL) + goto error; + + while (oparg--) { + Py_DECREF(POP()); + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + TARGET(SETUP_ANNOTATIONS) { + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + goto error; + } + /* check if __annotations__ in locals()... */ + if (PyDict_CheckExact(LOCALS())) { + ann_dict = _PyDict_GetItemWithError(LOCALS(), + &_Py_ID(__annotations__)); + if (ann_dict == NULL) { + if (_PyErr_Occurred(tstate)) { + goto error; + } + /* ...if not, create a new one */ + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { + goto error; + } + } + } + else { + /* do the same if locals() is not a dict */ + ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__)); + if (ann_dict == NULL) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto error; + } + _PyErr_Clear(tstate); + ann_dict = PyDict_New(); + if (ann_dict == NULL) { + goto error; + } + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err != 0) { + goto error; + } + } + else { + Py_DECREF(ann_dict); + } + } + DISPATCH(); + } + + TARGET(BUILD_CONST_KEY_MAP) { + PyObject *map; + PyObject *keys = TOP(); + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + goto error; + } + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + &PEEK(oparg + 1), 1, oparg); + if (map == NULL) { + goto error; + } + + Py_DECREF(POP()); + while (oparg--) { + Py_DECREF(POP()); + } + PUSH(map); + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + PyObject *update = POP(); + PyObject *dict = PEEK(oparg); + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + goto error; + } + Py_DECREF(update); + DISPATCH(); + } + + TARGET(DICT_MERGE) { + PyObject *update = POP(); + PyObject *dict = PEEK(oparg); + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + format_kwargs_error(tstate, PEEK(2 + oparg), update); + Py_DECREF(update); + goto error; + } + Py_DECREF(update); + PREDICT(CALL_FUNCTION_EX); + DISPATCH(); + } + + TARGET(MAP_ADD) { + PyObject *value = TOP(); + PyObject *key = SECOND(); + PyObject *map; + STACK_SHRINK(2); + map = PEEK(oparg); /* dict */ + assert(PyDict_CheckExact(map)); + /* map[key] = value */ + if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) { + goto error; + } + PREDICT(JUMP_BACKWARD_QUICK); + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + PREDICTED(LOAD_ATTR); + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *res = PyObject_GetAttr(owner, name); + if (res == NULL) { + goto error; + } + Py_DECREF(owner); + SET_TOP(res); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + + TARGET(LOAD_ATTR_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(LOAD_ATTR); + } + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *res; + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + assert(tp->tp_dictoffset < 0); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictValues *values = *_PyObject_ValuesPointer(owner); + DEOPT_IF(values == NULL, LOAD_ATTR); + res = values->values[cache->index]; + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + + TARGET(LOAD_ATTR_MODULE) { + assert(cframe.use_tracing == 0); + // shared with LOAD_METHOD_MODULE + PyObject *owner = TOP(); + PyObject *res; + LOAD_MODULE_ATTR_OR_METHOD(ATTR); + SET_TOP(res); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + + TARGET(LOAD_ATTR_WITH_HINT) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *res; + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(names, oparg); + uint16_t hint = cache->index; + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + res = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + res = ep->me_value; + } + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + + TARGET(LOAD_ATTR_SLOT) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *res; + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + char *addr = (char *)owner + cache->index; + res = *(PyObject **)addr; + DEOPT_IF(res == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(res); + SET_TOP(res); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); + DISPATCH(); + } + + DISPATCH(); + TARGET(STORE_ATTR_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(STORE_ATTR); + } + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictValues *values = *_PyObject_ValuesPointer(owner); + DEOPT_IF(values == NULL, STORE_ATTR); + STAT_INC(STORE_ATTR, hit); + Py_ssize_t index = cache->index; + STACK_SHRINK(1); + PyObject *value = POP(); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + DISPATCH(); + } + + TARGET(STORE_ATTR_WITH_HINT) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(names, oparg); + uint16_t hint = cache->index; + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *value, *old_value; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + STACK_SHRINK(1); + value = POP(); + ep->me_value = value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + STACK_SHRINK(1); + value = POP(); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = DICT_NEXT_VERSION(); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + DISPATCH(); + } + + TARGET(STORE_ATTR_SLOT) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyTypeObject *tp = Py_TYPE(owner); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + uint32_t type_version = read_u32(cache->version); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + char *addr = (char *)owner + cache->index; + STAT_INC(STORE_ATTR, hit); + STACK_SHRINK(1); + PyObject *value = POP(); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + DISPATCH(); + } + + TARGET(COMPARE_OP) { + PREDICTED(COMPARE_OP); + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + DISPATCH(); + } + + TARGET(COMPARE_OP_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *right = TOP(); + PyObject *left = SECOND(); + next_instr--; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(COMPARE_OP); + } + } + + TARGET(COMPARE_OP_FLOAT_JUMP) { + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false) + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + int when_to_jump_mask = cache->mask; + PyObject *right = TOP(); + PyObject *left = SECOND(); + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + int sign = (dleft > dright) - (dleft < dright); + DEOPT_IF(isnan(dleft), COMPARE_OP); + DEOPT_IF(isnan(dright), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + NEXTOPARG(); + STACK_SHRINK(2); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + assert(opcode == POP_JUMP_FORWARD_IF_FALSE || + opcode == POP_JUMP_BACKWARD_IF_FALSE || + opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_TRUE); + int jump = (9 << (sign + 1)) & when_to_jump_mask; + if (!jump) { + next_instr++; + } + else if (jump >= 8) { + assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_FALSE); + JUMPBY(1 - oparg); + CHECK_EVAL_BREAKER(); + } + else { + assert(opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_FORWARD_IF_FALSE); + JUMPBY(1 + oparg); + } + DISPATCH(); + } + + TARGET(COMPARE_OP_INT_JUMP) { + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false) + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + int when_to_jump_mask = cache->mask; + PyObject *right = TOP(); + PyObject *left = SECOND(); + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + int sign = (ileft > iright) - (ileft < iright); + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + NEXTOPARG(); + STACK_SHRINK(2); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + assert(opcode == POP_JUMP_FORWARD_IF_FALSE || + opcode == POP_JUMP_BACKWARD_IF_FALSE || + opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_TRUE); + int jump = (9 << (sign + 1)) & when_to_jump_mask; + if (!jump) { + next_instr++; + } + else if (jump >= 8) { + assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_FALSE); + JUMPBY(1 - oparg); + CHECK_EVAL_BREAKER(); + } + else { + assert(opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_FORWARD_IF_FALSE); + JUMPBY(1 + oparg); + } + DISPATCH(); + } + + TARGET(COMPARE_OP_STR_JUMP) { + assert(cframe.use_tracing == 0); + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false) + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + int when_to_jump_mask = cache->mask; + PyObject *right = TOP(); + PyObject *left = SECOND(); + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int res = _PyUnicode_Equal(left, right); + if (res < 0) { + goto error; + } + assert(oparg == Py_EQ || oparg == Py_NE); + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + NEXTOPARG(); + assert(opcode == POP_JUMP_FORWARD_IF_FALSE || + opcode == POP_JUMP_BACKWARD_IF_FALSE || + opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_TRUE); + STACK_SHRINK(2); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + int sign = 1 - res; + int jump = (9 << (sign + 1)) & when_to_jump_mask; + if (!jump) { + next_instr++; + } + else if (jump >= 8) { + assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || + opcode == POP_JUMP_BACKWARD_IF_FALSE); + JUMPBY(1 - oparg); + CHECK_EVAL_BREAKER(); + } + else { + assert(opcode == POP_JUMP_FORWARD_IF_TRUE || + opcode == POP_JUMP_FORWARD_IF_FALSE); + JUMPBY(1 + oparg); + } + DISPATCH(); + } + + TARGET(IS_OP) { + PyObject *right = POP(); + PyObject *left = TOP(); + int res = Py_Is(left, right) ^ oparg; + PyObject *b = res ? Py_True : Py_False; + Py_INCREF(b); + SET_TOP(b); + Py_DECREF(left); + Py_DECREF(right); + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + PyObject *right = POP(); + PyObject *left = POP(); + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) { + goto error; + } + PyObject *b = (res^oparg) ? Py_True : Py_False; + Py_INCREF(b); + PUSH(b); + DISPATCH(); + } + + TARGET(CHECK_EG_MATCH) { + PyObject *match_type = POP(); + if (check_except_star_type_valid(tstate, match_type) < 0) { + Py_DECREF(match_type); + goto error; + } + + PyObject *exc_value = TOP(); + PyObject *match = NULL, *rest = NULL; + int res = exception_group_match(exc_value, match_type, + &match, &rest); + Py_DECREF(match_type); + if (res < 0) { + goto error; + } + + if (match == NULL || rest == NULL) { + assert(match == NULL); + assert(rest == NULL); + goto error; + } + if (Py_IsNone(match)) { + PUSH(match); + Py_XDECREF(rest); + } + else { + /* Total or partial match - update the stack from + * [val] + * to + * [rest, match] + * (rest can be Py_None) + */ + + SET_TOP(rest); + PUSH(match); + PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); + Py_DECREF(exc_value); + } + DISPATCH(); + } + + TARGET(CHECK_EXC_MATCH) { + PyObject *right = POP(); + PyObject *left = TOP(); + assert(PyExceptionInstance_Check(left)); + if (check_except_type_valid(tstate, right) < 0) { + Py_DECREF(right); + goto error; + } + + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + PUSH(Py_NewRef(res ? Py_True : Py_False)); + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + PyObject *name = GETITEM(names, oparg); + PyObject *fromlist = POP(); + PyObject *level = TOP(); + PyObject *res; + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + SET_TOP(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + TARGET(IMPORT_STAR) { + PyObject *from = POP(), *locals; + int err; + if (_PyFrame_FastToLocalsWithError(frame) < 0) { + Py_DECREF(from); + goto error; + } + + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found during 'import *'"); + Py_DECREF(from); + goto error; + } + err = import_all_from(tstate, locals, from); + _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(from); + if (err != 0) + goto error; + DISPATCH(); + } + + TARGET(IMPORT_FROM) { + PyObject *name = GETITEM(names, oparg); + PyObject *from = TOP(); + PyObject *res; + res = import_from(tstate, from, name); + PUSH(res); + if (res == NULL) + goto error; + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + _PyCode_Warmup(frame->f_code); + JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK); + } + + TARGET(POP_JUMP_BACKWARD_IF_FALSE) { + PREDICTED(POP_JUMP_BACKWARD_IF_FALSE); + PyObject *cond = POP(); + if (Py_IsTrue(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + DISPATCH(); + } + if (Py_IsFalse(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) + ; + else if (err == 0) { + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + } + else + goto error; + DISPATCH(); + } + + TARGET(POP_JUMP_FORWARD_IF_FALSE) { + PREDICTED(POP_JUMP_FORWARD_IF_FALSE); + PyObject *cond = POP(); + if (Py_IsTrue(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + } + else if (Py_IsFalse(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + JUMPBY(oparg); + } + else { + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) + ; + else if (err == 0) { + JUMPBY(oparg); + } + else + goto error; + } + DISPATCH(); + } + + TARGET(POP_JUMP_BACKWARD_IF_TRUE) { + PyObject *cond = POP(); + if (Py_IsFalse(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + DISPATCH(); + } + if (Py_IsTrue(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) { + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + } + else if (err == 0) + ; + else + goto error; + DISPATCH(); + } + + TARGET(POP_JUMP_FORWARD_IF_TRUE) { + PyObject *cond = POP(); + if (Py_IsFalse(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + } + else if (Py_IsTrue(cond)) { + _Py_DECREF_NO_DEALLOC(cond); + JUMPBY(oparg); + } + else { + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err > 0) { + JUMPBY(oparg); + } + else if (err == 0) + ; + else + goto error; + } + DISPATCH(); + } + + TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) { + PyObject *value = POP(); + if (!Py_IsNone(value)) { + Py_DECREF(value); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + _Py_DECREF_NO_DEALLOC(value); + DISPATCH(); + } + + TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) { + PyObject *value = POP(); + if (!Py_IsNone(value)) { + JUMPBY(oparg); + } + Py_DECREF(value); + DISPATCH(); + } + + TARGET(POP_JUMP_BACKWARD_IF_NONE) { + PyObject *value = POP(); + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + } + else { + Py_DECREF(value); + } + DISPATCH(); + } + + TARGET(POP_JUMP_FORWARD_IF_NONE) { + PyObject *value = POP(); + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + JUMPBY(oparg); + } + else { + Py_DECREF(value); + } + DISPATCH(); + } + + TARGET(JUMP_IF_FALSE_OR_POP) { + PyObject *cond = TOP(); + int err; + if (Py_IsTrue(cond)) { + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(cond); + DISPATCH(); + } + if (Py_IsFalse(cond)) { + JUMPBY(oparg); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + if (err > 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + else if (err == 0) + JUMPBY(oparg); + else + goto error; + DISPATCH(); + } + + TARGET(JUMP_IF_TRUE_OR_POP) { + PyObject *cond = TOP(); + int err; + if (Py_IsFalse(cond)) { + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(cond); + DISPATCH(); + } + if (Py_IsTrue(cond)) { + JUMPBY(oparg); + DISPATCH(); + } + err = PyObject_IsTrue(cond); + if (err > 0) { + JUMPBY(oparg); + } + else if (err == 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + else + goto error; + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_QUICK) { + PREDICTED(JUMP_BACKWARD_QUICK); + assert(oparg < INSTR_OFFSET()); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(GET_LEN) { + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(TOP()); + if (len_i < 0) { + goto error; + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + goto error; + } + PUSH(len_o); + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + PyObject *names = POP(); + PyObject *type = POP(); + PyObject *subject = TOP(); + assert(PyTuple_CheckExact(names)); + PyObject *attrs = match_class(tstate, subject, type, oparg, names); + Py_DECREF(names); + Py_DECREF(type); + if (attrs) { + // Success! + assert(PyTuple_CheckExact(attrs)); + SET_TOP(attrs); + } + else if (_PyErr_Occurred(tstate)) { + // Error! + goto error; + } + else { + // Failure! + Py_INCREF(Py_None); + SET_TOP(Py_None); + } + Py_DECREF(subject); + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + PREDICT(POP_JUMP_FORWARD_IF_FALSE); + PREDICT(POP_JUMP_BACKWARD_IF_FALSE); + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + PyObject *subject = TOP(); + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + PyObject *res = match ? Py_True : Py_False; + Py_INCREF(res); + PUSH(res); + PREDICT(POP_JUMP_FORWARD_IF_FALSE); + PREDICT(POP_JUMP_BACKWARD_IF_FALSE); + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + // On successful match, PUSH(values). Otherwise, PUSH(None). + PyObject *keys = TOP(); + PyObject *subject = SECOND(); + PyObject *values_or_none = match_keys(tstate, subject, keys); + if (values_or_none == NULL) { + goto error; + } + PUSH(values_or_none); + DISPATCH(); + } + + TARGET(GET_ITER) { + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable = TOP(); + PyObject *iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + SET_TOP(iter); + if (iter == NULL) + goto error; + PREDICT(FOR_ITER); + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + /* before: [obj]; after [getiter(obj)] */ + PyObject *iterable = TOP(); + PyObject *iter; + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + Py_DECREF(iterable); + SET_TOP(NULL); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + goto error; + } + } + else if (!PyGen_CheckExact(iterable)) { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + SET_TOP(iter); + if (iter == NULL) + goto error; + } + PREDICT(LOAD_CONST); + DISPATCH(); + } + + TARGET(FOR_ITER) { + PREDICTED(FOR_ITER); + /* before: [iter]; after: [iter, iter()] *or* [] */ + PyObject *iter = TOP(); +#ifdef Py_STATS + extern int _PySpecialization_ClassifyIterator(PyObject *); + _py_stats.opcode_stats[FOR_ITER].specialization.failure++; + _py_stats.opcode_stats[FOR_ITER].specialization.failure_kinds[_PySpecialization_ClassifyIterator(iter)]++; +#endif + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + PREDICT(STORE_FAST); + PREDICT(UNPACK_SEQUENCE); + DISPATCH(); + } + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + else if (tstate->c_tracefunc != NULL) { + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + } + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + STACK_SHRINK(1); + Py_DECREF(iter); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(BEFORE_ASYNC_WITH) { + PyObject *mgr = TOP(); + PyObject *res; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + goto error; + } + PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + goto error; + } + SET_TOP(exit); + Py_DECREF(mgr); + res = _PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) + goto error; + PUSH(res); + PREDICT(GET_AWAITABLE); + DISPATCH(); + } + + TARGET(BEFORE_WITH) { + PyObject *mgr = TOP(); + PyObject *res; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); + } + goto error; + } + PyObject *exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + goto error; + } + SET_TOP(exit); + Py_DECREF(mgr); + res = _PyObject_CallNoArgs(enter); + Py_DECREF(enter); + if (res == NULL) { + goto error; + } + PUSH(res); + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + /* At the top of the stack are 4 values: + - TOP = exc_info() + - SECOND = previous exception + - THIRD: lasti of exception in exc_info() + - FOURTH: the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exit_func; + PyObject *exc, *val, *tb, *res; + + val = TOP(); + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + Py_XDECREF(tb); + assert(PyLong_Check(PEEK(3))); + exit_func = PEEK(4); + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) + goto error; + + PUSH(res); + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + PyObject *value = TOP(); + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + SET_TOP(exc_info->exc_value); + } + else { + Py_INCREF(Py_None); + SET_TOP(Py_None); + } + + Py_INCREF(value); + PUSH(value); + assert(PyExceptionInstance_Check(value)); + exc_info->exc_value = value; + + DISPATCH(); + } + + TARGET(LOAD_METHOD) { + PREDICTED(LOAD_METHOD); + /* Designed to work in tandem with PRECALL. */ + PyObject *name = GETITEM(names, oparg); + PyObject *obj = TOP(); + PyObject *meth = NULL; + + int meth_found = _PyObject_GetMethod(obj, name, &meth); + + if (meth == NULL) { + /* Most likely attribute wasn't found. */ + goto error; + } + + if (meth_found) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + + meth | self | arg1 | ... | argN + */ + SET_TOP(meth); + PUSH(obj); // self + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + PRECALL that it's not a method call. + + NULL | meth | arg1 | ... | argN + */ + SET_TOP(NULL); + Py_DECREF(obj); + PUSH(meth); + } + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(LOAD_METHOD_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_LoadMethod(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_METHOD, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(LOAD_METHOD); + } + } + + TARGET(LOAD_METHOD_WITH_VALUES) { + /* LOAD_METHOD, with cached method object */ + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + assert(type_version != 0); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_METHOD); + assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self); + DEOPT_IF(dict != NULL, LOAD_METHOD); + PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; + DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + read_u32(cache->keys_version), LOAD_METHOD); + STAT_INC(LOAD_METHOD, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + Py_INCREF(res); + SET_TOP(res); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(LOAD_METHOD_WITH_DICT) { + /* LOAD_METHOD, with a dict + Can be either a managed dict, or a tp_dictoffset offset.*/ + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + + DEOPT_IF(self_cls->tp_version_tag != read_u32(cache->type_version), + LOAD_METHOD); + /* Treat index as a signed 16 bit value */ + int dictoffset = *(int16_t *)&cache->dict_offset; + PyDictObject **dictptr = (PyDictObject**)(((char *)self)+dictoffset); + assert( + dictoffset == MANAGED_DICT_OFFSET || + (dictoffset == self_cls->tp_dictoffset && dictoffset > 0) + ); + PyDictObject *dict = *dictptr; + DEOPT_IF(dict == NULL, LOAD_METHOD); + DEOPT_IF(dict->ma_keys->dk_version != read_u32(cache->keys_version), + LOAD_METHOD); + STAT_INC(LOAD_METHOD, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + Py_INCREF(res); + SET_TOP(res); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(LOAD_METHOD_NO_DICT) { + assert(cframe.use_tracing == 0); + PyObject *self = TOP(); + PyTypeObject *self_cls = Py_TYPE(self); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + uint32_t type_version = read_u32(cache->type_version); + DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_METHOD); + assert(self_cls->tp_dictoffset == 0); + STAT_INC(LOAD_METHOD, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)); + Py_INCREF(res); + SET_TOP(res); + PUSH(self); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(LOAD_METHOD_MODULE) { + /* LOAD_METHOD, for module methods */ + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *res; + LOAD_MODULE_ATTR_OR_METHOD(METHOD); + SET_TOP(NULL); + Py_DECREF(owner); + PUSH(res); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(LOAD_METHOD_CLASS) { + /* LOAD_METHOD, for class methods */ + assert(cframe.use_tracing == 0); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; + + PyObject *cls = TOP(); + DEOPT_IF(!PyType_Check(cls), LOAD_METHOD); + uint32_t type_version = read_u32(cache->type_version); + DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, + LOAD_METHOD); + assert(type_version != 0); + + STAT_INC(LOAD_METHOD, hit); + PyObject *res = read_obj(cache->descr); + assert(res != NULL); + Py_INCREF(res); + SET_TOP(NULL); + Py_DECREF(cls); + PUSH(res); + JUMPBY(INLINE_CACHE_ENTRIES_LOAD_METHOD); + DISPATCH(); + } + + TARGET(PRECALL) { + PREDICTED(PRECALL); + /* Designed to work in tamdem with LOAD_METHOD. */ + /* `meth` is NULL when LOAD_METHOD thinks that it's not + a method call. + + Stack layout: + + ... | NULL | callable | arg1 | ... | argN + ^- TOP() + ^- (-oparg) + ^- (-oparg-1) + ^- (-oparg-2) + + `callable` will be POPed by call_function. + NULL will will be POPed manually later. + If `meth` isn't NULL, it's a method call. Stack layout: + + ... | method | self | arg1 | ... | argN + ^- TOP() + ^- (-oparg) + ^- (-oparg-1) + ^- (-oparg-2) + + `self` and `method` will be POPed by call_function. + We'll be passing `oparg + 1` to call_function, to + make it accept the `self` as a first argument. + */ + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + /* Move ownership of reference from stack to call_shape + * and make sure that NULL is cleared from stack */ + PyObject *function = PEEK(nargs + 1); + if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { + PyObject *meth = ((PyMethodObject *)function)->im_func; + PyObject *self = ((PyMethodObject *)function)->im_self; + Py_INCREF(meth); + Py_INCREF(self); + PEEK(oparg+1) = self; + PEEK(oparg+2) = meth; + Py_DECREF(function); + } + JUMPBY(INLINE_CACHE_ENTRIES_PRECALL); + DISPATCH(); + } + + TARGET(PRECALL_BOUND_METHOD) { + DEOPT_IF(is_method(stack_pointer, oparg), PRECALL); + PyObject *function = PEEK(oparg + 1); + DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, PRECALL); + STAT_INC(PRECALL, hit); + PyObject *meth = ((PyMethodObject *)function)->im_func; + PyObject *self = ((PyMethodObject *)function)->im_self; + Py_INCREF(meth); + Py_INCREF(self); + PEEK(oparg + 1) = self; + PEEK(oparg + 2) = meth; + Py_DECREF(function); + JUMPBY(INLINE_CACHE_ENTRIES_PRECALL); + DISPATCH(); + } + + TARGET(PRECALL_PYFUNC) { + int nargs = oparg + is_method(stack_pointer, oparg); + PyObject *function = PEEK(nargs + 1); + DEOPT_IF(Py_TYPE(function) != &PyFunction_Type, PRECALL); + STAT_INC(PRECALL, hit); + JUMPBY(INLINE_CACHE_ENTRIES_PRECALL); + DISPATCH(); + } + + TARGET(KW_NAMES) { + assert(call_shape.kwnames == NULL); + assert(oparg < PyTuple_GET_SIZE(consts)); + call_shape.kwnames = GETITEM(consts, oparg); + DISPATCH(); + } + + TARGET(CALL) { + int is_meth; + call_function: + is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + int positional_args = total_args - KWNAMES_LEN(); + // Check if the call can be inlined or not + if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function); + STACK_SHRINK(total_args); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)function, locals, + stack_pointer, positional_args, call_shape.kwnames + ); + call_shape.kwnames = NULL; + STACK_SHRINK(2-is_meth); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + /* Callable is not a normal Python function */ + PyObject *res; + if (cframe.use_tracing) { + res = trace_call_function( + tstate, function, stack_pointer-total_args, + positional_args, call_shape.kwnames); + } + else { + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + call_shape.kwnames); + } + call_shape.kwnames = NULL; + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(function); + /* Clear the stack */ + STACK_SHRINK(total_args); + for (int i = 0; i < total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + STACK_SHRINK(2-is_meth); + PUSH(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_ADAPTIVE) { + _PyPrecallCache *cache = (_PyPrecallCache *)next_instr; + if (cache->counter == 0) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Precall(callable, next_instr, nargs, + call_shape.kwnames, oparg); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(PRECALL, deferred); + cache->counter--; + JUMP_TO_INSTRUCTION(PRECALL); + } + } + + TARGET(CALL_ADAPTIVE) { + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call(callable, next_instr, nargs, + call_shape.kwnames); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + goto call_function; + } + } + + TARGET(CALL_PY_EXACT_ARGS) { + assert(call_shape.kwnames == NULL); + DEOPT_IF(tstate->interp->eval_frame, CALL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + int is_meth = is_method(stack_pointer, oparg); + int argcount = oparg + is_meth; + PyObject *callable = PEEK(argcount + 1); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != argcount, CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_Push(tstate, func); + if (new_frame == NULL) { + goto error; + } + CALL_STAT_INC(inlined_py_calls); + STACK_SHRINK(argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = stack_pointer[i]; + } + for (int i = argcount; i < code->co_nlocalsplus; i++) { + new_frame->localsplus[i] = NULL; + } + STACK_SHRINK(2-is_meth); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + frame = cframe.current_frame = new_frame; + goto start_frame; + } + + TARGET(CALL_PY_WITH_DEFAULTS) { + assert(call_shape.kwnames == NULL); + DEOPT_IF(tstate->interp->eval_frame, CALL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + int is_meth = is_method(stack_pointer, oparg); + int argcount = oparg + is_meth; + PyObject *callable = PEEK(argcount + 1); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(argcount > code->co_argcount, CALL); + int minargs = cache->min_args; + DEOPT_IF(argcount < minargs, CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_Push(tstate, func); + if (new_frame == NULL) { + goto error; + } + CALL_STAT_INC(inlined_py_calls); + STACK_SHRINK(argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = stack_pointer[i]; + } + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, + i - minargs); + Py_INCREF(def); + new_frame->localsplus[i] = def; + } + for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) { + new_frame->localsplus[i] = NULL; + } + STACK_SHRINK(2-is_meth); + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + frame = cframe.current_frame = new_frame; + goto start_frame; + } + + TARGET(PRECALL_NO_KW_TYPE_1) { + assert(call_shape.kwnames == NULL); + assert(cframe.use_tracing == 0); + assert(oparg == 1); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); + PyObject *obj = TOP(); + PyObject *callable = SECOND(); + DEOPT_IF(callable != (PyObject *)&PyType_Type, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyObject *res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(callable); + Py_DECREF(obj); + STACK_SHRINK(2); + SET_TOP(res); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_STR_1) { + assert(call_shape.kwnames == NULL); + assert(cframe.use_tracing == 0); + assert(oparg == 1); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); + PyObject *callable = PEEK(2); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyObject *arg = TOP(); + PyObject *res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); + STACK_SHRINK(2); + SET_TOP(res); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_TUPLE_1) { + assert(call_shape.kwnames == NULL); + assert(oparg == 1); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); + PyObject *callable = PEEK(2); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyObject *arg = TOP(); + PyObject *res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); + STACK_SHRINK(2); + SET_TOP(res); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_BUILTIN_CLASS) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + int kwnames_len = KWNAMES_LEN(); + PyObject *callable = PEEK(total_args + 1); + DEOPT_IF(!PyType_Check(callable), PRECALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + STACK_SHRINK(total_args); + PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, + total_args-kwnames_len, call_shape.kwnames); + call_shape.kwnames = NULL; + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + Py_DECREF(tp); + STACK_SHRINK(1-is_meth); + SET_TOP(res); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_BUILTIN_O) { + assert(cframe.use_tracing == 0); + /* Builtin METH_O functions */ + assert(call_shape.kwnames == NULL); + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + DEOPT_IF(total_args != 1, PRECALL); + PyObject *callable = PEEK(total_args + 1); + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + PyObject *arg = TOP(); + PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(arg); + Py_DECREF(callable); + STACK_SHRINK(2-is_meth); + SET_TOP(res); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_BUILTIN_FAST) { + assert(cframe.use_tracing == 0); + /* Builtin METH_FASTCALL functions, without keywords */ + assert(call_shape.kwnames == NULL); + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *callable = PEEK(total_args + 1); + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, + PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + STACK_SHRINK(total_args); + /* res = func(self, args, nargs) */ + PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + stack_pointer, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + STACK_SHRINK(2-is_meth); + PUSH(res); + Py_DECREF(callable); + if (res == NULL) { + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(cframe.use_tracing == 0); + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *callable = PEEK(total_args + 1); + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != + (METH_FASTCALL | METH_KEYWORDS), PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + STACK_SHRINK(total_args); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + PyObject *res = cfunc( + PyCFunction_GET_SELF(callable), + stack_pointer, + total_args - KWNAMES_LEN(), + call_shape.kwnames + ); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + call_shape.kwnames = NULL; + + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + STACK_SHRINK(2-is_meth); + PUSH(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_LEN) { + assert(cframe.use_tracing == 0); + assert(call_shape.kwnames == NULL); + /* len(o) */ + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + DEOPT_IF(total_args != 1, PRECALL); + PyObject *callable = PEEK(total_args + 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + DEOPT_IF(callable != interp->callable_cache.len, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyObject *arg = TOP(); + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + goto error; + } + PyObject *res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACK_SHRINK(2-is_meth); + SET_TOP(res); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_ISINSTANCE) { + assert(cframe.use_tracing == 0); + assert(call_shape.kwnames == NULL); + /* isinstance(o, o2) */ + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *callable = PEEK(total_args + 1); + DEOPT_IF(total_args != 2, PRECALL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + DEOPT_IF(callable != interp->callable_cache.isinstance, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyObject *cls = POP(); + PyObject *inst = TOP(); + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + Py_DECREF(cls); + goto error; + } + PyObject *res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACK_SHRINK(2-is_meth); + SET_TOP(res); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_LIST_APPEND) { + assert(cframe.use_tracing == 0); + assert(call_shape.kwnames == NULL); + assert(oparg == 1); + PyObject *callable = PEEK(3); + PyInterpreterState *interp = _PyInterpreterState_GET(); + DEOPT_IF(callable != interp->callable_cache.list_append, PRECALL); + PyObject *list = SECOND(); + DEOPT_IF(!PyList_Check(list), PRECALL); + STAT_INC(PRECALL, hit); + // PRECALL + CALL + POP_TOP + JUMPBY(INLINE_CACHE_ENTRIES_PRECALL + 1 + INLINE_CACHE_ENTRIES_CALL + 1); + assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); + PyObject *arg = POP(); + if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { + goto error; + } + STACK_SHRINK(2); + Py_DECREF(list); + Py_DECREF(callable); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_O) { + assert(call_shape.kwnames == NULL); + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyMethodDescrObject *callable = + (PyMethodDescrObject *)PEEK(total_args + 1); + DEOPT_IF(total_args != 2, PRECALL); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); + PyMethodDef *meth = callable->d_method; + DEOPT_IF(meth->ml_flags != METH_O, PRECALL); + PyObject *arg = TOP(); + PyObject *self = SECOND(); + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + PyObject *res = cfunc(self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + STACK_SHRINK(oparg + 1); + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyMethodDescrObject *callable = + (PyMethodDescrObject *)PEEK(total_args + 1); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); + PyMethodDef *meth = callable->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), PRECALL); + PyTypeObject *d_type = callable->d_common.d_type; + PyObject *self = PEEK(total_args); + DEOPT_IF(!Py_IS_TYPE(self, d_type), PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + int nargs = total_args-1; + STACK_SHRINK(nargs); + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), + call_shape.kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + call_shape.kwnames = NULL; + + /* Free the arguments. */ + for (int i = 0; i < nargs; i++) { + Py_DECREF(stack_pointer[i]); + } + Py_DECREF(self); + STACK_SHRINK(2-is_meth); + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + assert(call_shape.kwnames == NULL); + assert(oparg == 0 || oparg == 1); + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + DEOPT_IF(total_args != 1, PRECALL); + PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); + PyMethodDef *meth = callable->d_method; + PyObject *self = TOP(); + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + PyObject *res = cfunc(self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + STACK_SHRINK(oparg + 1); + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + assert(call_shape.kwnames == NULL); + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyMethodDescrObject *callable = + (PyMethodDescrObject *)PEEK(total_args + 1); + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); + PyMethodDef *meth = callable->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, PRECALL); + PyObject *self = PEEK(total_args); + DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), PRECALL); + STAT_INC(PRECALL, hit); + SKIP_CALL(); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args-1; + STACK_SHRINK(nargs); + PyObject *res = cfunc(self, stack_pointer, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < nargs; i++) { + Py_DECREF(stack_pointer[i]); + } + Py_DECREF(self); + STACK_SHRINK(2-is_meth); + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + PREDICTED(CALL_FUNCTION_EX); + PyObject *func, *callargs, *kwargs = NULL, *result; + if (oparg & 0x01) { + kwargs = POP(); + if (!PyDict_CheckExact(kwargs)) { + PyObject *d = PyDict_New(); + if (d == NULL) + goto error; + if (_PyDict_MergeEx(d, kwargs, 2) < 0) { + Py_DECREF(d); + format_kwargs_error(tstate, SECOND(), kwargs); + Py_DECREF(kwargs); + goto error; + } + Py_DECREF(kwargs); + kwargs = d; + } + assert(PyDict_CheckExact(kwargs)); + } + callargs = POP(); + func = TOP(); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + Py_DECREF(callargs); + goto error; + } + Py_SETREF(callargs, PySequence_Tuple(callargs)); + if (callargs == NULL) { + goto error; + } + } + assert(PyTuple_CheckExact(callargs)); + + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + + STACK_SHRINK(1); + assert(TOP() == NULL); + SET_TOP(result); + if (result == NULL) { + goto error; + } + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + PyObject *codeobj = POP(); + PyFunctionObject *func = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + + Py_DECREF(codeobj); + if (func == NULL) { + goto error; + } + + if (oparg & 0x08) { + assert(PyTuple_CheckExact(TOP())); + func->func_closure = POP(); + } + if (oparg & 0x04) { + assert(PyTuple_CheckExact(TOP())); + func->func_annotations = POP(); + } + if (oparg & 0x02) { + assert(PyDict_CheckExact(TOP())); + func->func_kwdefaults = POP(); + } + if (oparg & 0x01) { + assert(PyTuple_CheckExact(TOP())); + func->func_defaults = POP(); + } + + PUSH((PyObject *)func); + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(frame->f_func); + if (gen == NULL) { + goto error; + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallTstate(tstate); + if (!frame->is_entry) { + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = cframe.current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + goto resume_frame; + } + /* Make sure that frame is in a valid state */ + frame->stacktop = 0; + frame->f_locals = NULL; + Py_INCREF(frame->f_func); + Py_INCREF(frame->f_code); + /* Restore previous cframe and return. */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; + assert(tstate->cframe->current_frame == frame->previous); + assert(!_PyErr_Occurred(tstate)); + return (PyObject *)gen; + } + + TARGET(BUILD_SLICE) { + PyObject *start, *stop, *step, *slice; + if (oparg == 3) + step = POP(); + else + step = NULL; + stop = POP(); + start = TOP(); + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + SET_TOP(slice); + if (slice == NULL) + goto error; + DISPATCH(); + } + + TARGET(FORMAT_VALUE) { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = POP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_NONE: conv_fn = NULL; break; + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + default: + _PyErr_Format(tstate, PyExc_SystemError, + "unexpected conversion flag %d", + which_conversion); + goto error; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn != NULL) { + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (result == NULL) { + goto error; + } + } + + PUSH(result); + DISPATCH(); + } + + TARGET(COPY) { + assert(oparg != 0); + PyObject *peek = PEEK(oparg); + Py_INCREF(peek); + PUSH(peek); + DISPATCH(); + } + + TARGET(BINARY_OP) { + PREDICTED(BINARY_OP); + PyObject *rhs = POP(); + PyObject *lhs = TOP(); + assert(0 <= oparg); + assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); + assert(binary_ops[oparg]); + PyObject *res = binary_ops[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + SET_TOP(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); + } + + TARGET(BINARY_OP_ADAPTIVE) { + assert(cframe.use_tracing == 0); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(BINARY_OP); + } + } + + TARGET(SWAP) { + assert(oparg != 0); + PyObject *top = TOP(); + SET_TOP(PEEK(oparg)); + PEEK(oparg) = top; + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + assert(oparg); + oparg <<= 8; + oparg |= _Py_OPARG(*next_instr); + // We might be tracing. To avoid breaking tracing guarantees in + // quickened instructions, always deoptimize the next opcode: + opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; + PRE_DISPATCH_GOTO(); + // CPython hasn't traced the following instruction historically + // (DO_TRACING would clobber our extended oparg anyways), so just + // skip our usual cframe.use_tracing check before dispatch. Also, + // make sure the next instruction isn't a RESUME, since that needs + // to trace properly (and shouldn't have an extended arg anyways): + assert(opcode != RESUME); + DISPATCH_GOTO(); + } + + TARGET(EXTENDED_ARG_QUICK) { + assert(cframe.use_tracing == 0); + assert(oparg); + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; + DISPATCH_GOTO(); + } + + TARGET(CACHE) { + Py_UNREACHABLE(); + } + +#if USE_COMPUTED_GOTOS + TARGET_DO_TRACING: +#else + case DO_TRACING: +#endif + { + assert(cframe.use_tracing); + assert(tstate->tracing == 0); + if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { + int instr_prev = _PyInterpreterFrame_LASTI(frame); + frame->prev_instr = next_instr; + TRACING_NEXTOPARG(); + if (opcode == RESUME) { + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + /* Call tracing */ + TRACE_FUNCTION_ENTRY(); + DTRACE_FUNCTION_ENTRY(); + } + else { + /* line-by-line tracing support */ + if (PyDTrace_LINE_ENABLED()) { + maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); + } + + if (cframe.use_tracing && + tstate->c_tracefunc != NULL && !tstate->tracing) { + int err; + /* see maybe_call_line_trace() + for expository comments */ + _PyFrame_SetStackPointer(frame, stack_pointer); + + err = maybe_call_line_trace(tstate->c_tracefunc, + tstate->c_traceobj, + tstate, frame, instr_prev); + // Reload possibly changed frame fields: + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; + // next_instr is only reloaded if tracing *does not* raise. + // This is consistent with the behavior of older Python + // versions. If a trace function sets a new f_lineno and + // *then* raises, we use the *old* location when searching + // for an exception handler, displaying the traceback, and + // so on: + if (err) { + // next_instr wasn't incremented at the start of this + // instruction. Increment it before handling the error, + // so that it looks the same as a "normal" instruction: + next_instr++; + goto error; + } + // Reload next_instr. Don't increment it, though, since + // we're going to re-dispatch to the "true" instruction now: + next_instr = frame->prev_instr; + } + } + } + TRACING_NEXTOPARG(); + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From opcode.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = _Py_OPCODE(*next_instr); + fprintf(stderr, "XXX lineno: %d, opcode: %d\n", + _PyInterpreterFrame_GetLine(frame), opcode); + _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode"); + goto error; + + } /* End instructions */ + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + +/* Specialization misses */ + +miss: + { + STAT_INC(opcode, miss); + opcode = _PyOpcode_Deopt[opcode]; + STAT_INC(opcode, miss); + /* The counter is always the first cache entry: */ + _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr; + *counter -= 1; + if (*counter == 0) { + int adaptive_opcode = _PyOpcode_Adaptive[opcode]; + assert(adaptive_opcode); + _Py_SET_OPCODE(next_instr[-1], adaptive_opcode); + STAT_INC(opcode, deopt); + *counter = adaptive_counter_start(); + } + next_instr--; + DISPATCH_GOTO(); + } + +binary_subscr_dict_error: + { + PyObject *sub = POP(); + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(sub); + goto error; + } + +unbound_local_error: + { + format_exc_check_arg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg) + ); + goto error; + } + +error: + call_shape.kwnames = NULL; + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + + if (tstate->c_tracefunc != NULL) { + /* Make sure state is set to FRAME_UNWINDING for tracing */ + call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, + tstate, frame); + } + +exception_unwind: + { + /* We can't use frame->f_lasti here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + PyObject **stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyObject *o = POP(); + Py_XDECREF(o); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_UNWIND(); + DTRACE_FUNCTION_EXIT(); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + PyObject **new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyObject *v = POP(); + Py_XDECREF(v); + } + PyObject *exc, *val, *tb; + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(lasti); + } + _PyErr_Fetch(tstate, &exc, &val, &tb); + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + _PyErr_NormalizeException(tstate, &exc, &val, &tb); + if (tb != NULL) + PyException_SetTraceback(val, tb); + else + PyException_SetTraceback(val, Py_None); + Py_XDECREF(tb); + Py_XDECREF(exc); + PUSH(val); + JUMPTO(handler); + /* Resume normal execution */ + DISPATCH(); + } + } + +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallTstate(tstate); + if (frame->is_entry) { + /* Restore previous cframe and exit */ + tstate->cframe = cframe.previous; + tstate->cframe->use_tracing = cframe.use_tracing; + assert(tstate->cframe->current_frame == frame->previous); + return NULL; + } + frame = cframe.current_frame = pop_frame(tstate, frame); + +resume_with_error: + SET_LOCALS_FROM_FRAME(); + goto error; + +} + + +PyObject * +_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames) +{ + /* _PyEvalFramePushAndInit consumes the references + * to func and all its arguments */ + Py_INCREF(func); + for (size_t i = 0; i < argcount; i++) { + Py_INCREF(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_INCREF(args[i+argcount]); + } + } + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + tstate, func, locals, args, argcount, kwnames); + if (frame == NULL) { + return NULL; + } + PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0); + assert( + _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) || + _PyFrame_GetStackPointer(frame) == frame->localsplus + ); + _PyEvalFrameClearAndPop(tstate, frame); + return retval; +} diff --git a/lecture20/cpython/3.11.0rc2/opcode.h b/lecture20/cpython/3.11.0rc2/opcode.h @@ -0,0 +1,236 @@ +// Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py + +#ifndef Py_OPCODE_H +#define Py_OPCODE_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* Instruction opcodes for compiled code */ +#define CACHE 0 +#define POP_TOP 1 +#define PUSH_NULL 2 +#define NOP 9 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define BINARY_SUBSCR 25 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define CHECK_EXC_MATCH 36 +#define CHECK_EG_MATCH 37 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define LOAD_ASSERTION_ERROR 74 +#define RETURN_GENERATOR 75 +#define LIST_TO_TUPLE 82 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 +#define SETUP_ANNOTATIONS 85 +#define YIELD_VALUE 86 +#define ASYNC_GEN_WRAP 87 +#define PREP_RERAISE_STAR 88 +#define POP_EXCEPT 89 +#define HAVE_ARGUMENT 90 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define SWAP 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define JUMP_IF_FALSE_OR_POP 111 +#define JUMP_IF_TRUE_OR_POP 112 +#define POP_JUMP_FORWARD_IF_FALSE 114 +#define POP_JUMP_FORWARD_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define BINARY_OP 122 +#define SEND 123 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define POP_JUMP_FORWARD_IF_NOT_NONE 128 +#define POP_JUMP_FORWARD_IF_NONE 129 +#define RAISE_VARARGS 130 +#define GET_AWAITABLE 131 +#define MAKE_FUNCTION 132 +#define BUILD_SLICE 133 +#define JUMP_BACKWARD_NO_INTERRUPT 134 +#define MAKE_CELL 135 +#define LOAD_CLOSURE 136 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define JUMP_BACKWARD 140 +#define CALL_FUNCTION_EX 142 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define LOAD_CLASSDEREF 148 +#define COPY_FREE_VARS 149 +#define RESUME 151 +#define MATCH_CLASS 152 +#define FORMAT_VALUE 155 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define LOAD_METHOD 160 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define PRECALL 166 +#define CALL 171 +#define KW_NAMES 172 +#define POP_JUMP_BACKWARD_IF_NOT_NONE 173 +#define POP_JUMP_BACKWARD_IF_NONE 174 +#define POP_JUMP_BACKWARD_IF_FALSE 175 +#define POP_JUMP_BACKWARD_IF_TRUE 176 +#define BINARY_OP_ADAPTIVE 3 +#define BINARY_OP_ADD_FLOAT 4 +#define BINARY_OP_ADD_INT 5 +#define BINARY_OP_ADD_UNICODE 6 +#define BINARY_OP_INPLACE_ADD_UNICODE 7 +#define BINARY_OP_MULTIPLY_FLOAT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_SUBTRACT_INT 16 +#define BINARY_SUBSCR_ADAPTIVE 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_ADAPTIVE 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define COMPARE_OP_ADAPTIVE 26 +#define COMPARE_OP_FLOAT_JUMP 27 +#define COMPARE_OP_INT_JUMP 28 +#define COMPARE_OP_STR_JUMP 29 +#define EXTENDED_ARG_QUICK 34 +#define JUMP_BACKWARD_QUICK 38 +#define LOAD_ATTR_ADAPTIVE 39 +#define LOAD_ATTR_INSTANCE_VALUE 40 +#define LOAD_ATTR_MODULE 41 +#define LOAD_ATTR_SLOT 42 +#define LOAD_ATTR_WITH_HINT 43 +#define LOAD_CONST__LOAD_FAST 44 +#define LOAD_FAST__LOAD_CONST 45 +#define LOAD_FAST__LOAD_FAST 46 +#define LOAD_GLOBAL_ADAPTIVE 47 +#define LOAD_GLOBAL_BUILTIN 48 +#define LOAD_GLOBAL_MODULE 55 +#define LOAD_METHOD_ADAPTIVE 56 +#define LOAD_METHOD_CLASS 57 +#define LOAD_METHOD_MODULE 58 +#define LOAD_METHOD_NO_DICT 59 +#define LOAD_METHOD_WITH_DICT 62 +#define LOAD_METHOD_WITH_VALUES 63 +#define PRECALL_ADAPTIVE 64 +#define PRECALL_BOUND_METHOD 65 +#define PRECALL_BUILTIN_CLASS 66 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 67 +#define PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 72 +#define PRECALL_NO_KW_BUILTIN_FAST 73 +#define PRECALL_NO_KW_BUILTIN_O 76 +#define PRECALL_NO_KW_ISINSTANCE 77 +#define PRECALL_NO_KW_LEN 78 +#define PRECALL_NO_KW_LIST_APPEND 79 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 81 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 113 +#define PRECALL_NO_KW_STR_1 121 +#define PRECALL_NO_KW_TUPLE_1 127 +#define PRECALL_NO_KW_TYPE_1 141 +#define PRECALL_PYFUNC 143 +#define RESUME_QUICK 150 +#define STORE_ATTR_ADAPTIVE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 167 +#define STORE_SUBSCR_ADAPTIVE 168 +#define STORE_SUBSCR_DICT 169 +#define STORE_SUBSCR_LIST_INT 170 +#define UNPACK_SEQUENCE_ADAPTIVE 177 +#define UNPACK_SEQUENCE_LIST 178 +#define UNPACK_SEQUENCE_TUPLE 179 +#define UNPACK_SEQUENCE_TWO_TUPLE 180 +#define DO_TRACING 255 + +#define HAS_CONST(op) (false\ + || ((op) == 100) \ + || ((op) == 172) \ + ) + +#define NB_ADD 0 +#define NB_AND 1 +#define NB_FLOOR_DIVIDE 2 +#define NB_LSHIFT 3 +#define NB_MATRIX_MULTIPLY 4 +#define NB_MULTIPLY 5 +#define NB_REMAINDER 6 +#define NB_OR 7 +#define NB_POWER 8 +#define NB_RSHIFT 9 +#define NB_SUBTRACT 10 +#define NB_TRUE_DIVIDE 11 +#define NB_XOR 12 +#define NB_INPLACE_ADD 13 +#define NB_INPLACE_AND 14 +#define NB_INPLACE_FLOOR_DIVIDE 15 +#define NB_INPLACE_LSHIFT 16 +#define NB_INPLACE_MATRIX_MULTIPLY 17 +#define NB_INPLACE_MULTIPLY 18 +#define NB_INPLACE_REMAINDER 19 +#define NB_INPLACE_OR 20 +#define NB_INPLACE_POWER 21 +#define NB_INPLACE_RSHIFT 22 +#define NB_INPLACE_SUBTRACT 23 +#define NB_INPLACE_TRUE_DIVIDE 24 +#define NB_INPLACE_XOR 25 + +#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) + +/* Reserve some bytecodes for internal use in the compiler. + * The value of 240 is arbitrary. */ +#define IS_ARTIFICIAL(op) ((op) > 240) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_H */ diff --git a/lecture20/global_local.py b/lecture20/global_local.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +def f(x): + l0 = x # l0 and x are local names + l1 = g # g is a global name + print(f'id(x) = {id(x)}') + print(f'id(g) = {id(g)}') + print(f'Local vars in f(x): {locals()}') + print(f'Global vars in f(x): {globals()}') + + +g = 42 # global name +# local and global scopes are identical in `__main__`: +# print(f'Local vars in `__main__`: {locals()}') +# print(f'Global vars in `__main__`: {globals()}') +f(g) diff --git a/lecture20/nonlocal.py b/lecture20/nonlocal.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import dis + + +def f_local_unbound(x): + def closure(y): + x -= y # shadows `x` in the outer function + return y + + return closure + + +def f_local_bound(x): + def closure(y): + y = x # `x` does not shadow itself because it is not redefined in local scope + return y + + return closure + +def f_nonlocal(x): + def closure(y): + nonlocal x + x -= y + return x + + return closure + + +def f_global(x): + def closure(y): + global x + x -= y + return x + + return closure + + +def disassemble(f, val=0): + c = f(val) + # Tuple containing names of local variables referenced in nested functions + print(f"{'Cell vars `' + f.__name__ + '`:':<32}{f.__code__.co_cellvars}") + # Tuple containing names of local variables in outer function + print(f"{'Local vars `' + f.__name__ + '`:':<32}{f.__code__.co_varnames}") + # Tuple containing names of local variables in closure + print(f"{'Local vars `' + c.__name__ + '`:':<32}{c.__code__.co_varnames}") + + print('Disassembly:') + dis.dis(c) + print('') + + +disassemble(f_local_unbound) +disassemble(f_local_bound) +disassemble(f_nonlocal) +disassemble(f_global) diff --git a/lecture21/presidential/.gitignore b/lecture21/presidential/.gitignore @@ -0,0 +1 @@ +*.sqlite diff --git a/lecture21/presidential/candidates.txt b/lecture21/presidential/candidates.txt @@ -0,0 +1,18 @@ +id|first_name|last_name|middle_name|party +33|Joseph|Biden||D +36|Samuel|Brownback||R +34|Hillary|Clinton|R.|D +39|Christopher|Dodd|J.|D +26|John|Edwards||D +22|Rudolph|Giuliani||R +24|Mike|Gravel||D +16|Mike|Huckabee||R +30|Duncan|Hunter||R +31|Dennis|Kucinich||D +37|John|McCain||R +20|Barack|Obama||D +32|Ron|Paul||R +29|Bill|Richardson||D +35|Mitt|Romney||R +38|Tom|Tancredo||R +41|Fred|Thompson|D.|R diff --git a/lecture21/presidential/contributors.txt b/lecture21/presidential/contributors.txt @@ -0,0 +1,176 @@ +id|last_name|first_name|middle_name|street_1|street_2|city|state|zip|amount|date|candidate_id +|Agee|Steven||549 Laurel Branch Road||Floyd|VA|24091|500.00|2007-06-30|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|250.00|2007-05-16|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|50.00|2007-06-18|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|100.00|2007-06-21|16 +|Akin|Charles||10187 Sugar Creek Road||Bentonville|AR|72712|100.00|2007-06-16|16 +|Akin|Mike||181 Baywood Lane||Monticello|AR|71655|1500.00|2007-05-18|16 +|Akin|Rebecca||181 Baywood Lane||Monticello|AR|71655|500.00|2007-05-18|16 +|Aldridge|Brittni||808 Capitol Square Place, SW||Washington|DC|20024|250.00|2007-06-06|16 +|Allen|John D.||1052 Cannon Mill Drive||North Augusta|SC|29860|1000.00|2007-06-11|16 +|Allen|John D.||1052 Cannon Mill Drive||North Augusta|SC|29860|1300.00|2007-06-29|16 +|Allison|John W.||P.O. Box 1089||Conway|AR|72033|1000.00|2007-05-18|16 +|Allison|Rebecca||3206 Summit Court||Little Rock|AR|72227|1000.00|2007-04-25|16 +|Allison|Rebecca||3206 Summit Court||Little Rock|AR|72227|200.00|2007-06-12|16 +|Altes|R.D.||8600 Moody Road||Fort Smith|AR|72903|2300.00|2007-06-21|16 +|Andres|Dale||1160 Glen Oaks Drive||West Des Moines|IA|50266|250.00|2007-06-06|16 +|Anthony|John||211 Long Island Drive||Hot Springs|AR|71913|2300.00|2007-06-12|16 +|Arbogast|Robert||12900 State Route 56 SE||Mount Sterling|OH|43143|500.00|2007-04-08|16 +|Arbogast|Robert||12900 State Route 56 SE||Mount Sterling|OH|43143|100.00|2007-06-22|16 +|Ardle|William||412 Dakota Avenue||Springfield|OH|45504|50.00|2007-06-28|16 +|Atiq|Omar||7200 S Hazel Street||Pine Bluff|AR|71603|1000.00|2007-05-18|16 +|Atiq|Omar||7200 S Hazel Street||Pine Bluff|AR|71603|1000.00|2007-06-27|16 +|Baker|David||2550 Adamsbrooke Drive||Conway|AR|72034|2300.00|2007-04-11|16 +|Bancroft|David||2934 Broderick Street||San Francisco|CA|94123|250.00|2007-04-24|16 +|Banks|Charles||P.O. Box 251310||Little Rock|AR|72225|1000.00|2007-05-14|16 +|Barbee|John||516 Kellyridge Drive||Apex|NC|27502|500.00|2007-05-23|16 +|Buckler|Steve||24351 Armada Dr||Dana Point|CA|926291306|50.00|2007-07-30|20 +|Buckler|Steve||24351 Armada Dr||Dana Point|CA|926291306|25.00|2007-08-16|20 +|Buckheit|Bruce||8904 KAREN DR||FAIRFAX|VA|220312731|100.00|2007-09-19|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|2300.00|2007-08-14|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|-2300.00|2007-08-14|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|4600.00|2007-08-14|20 +|Buck|Thomas||4206 Terrace Street||Kansas City|MO|64111|100.00|2007-09-25|20 +|Buck|Jay|K.|1855 Old Willow Rd Unit 322||Northfield|IL|600932918|200.00|2007-09-12|20 +|Buck|Blaine|M|45 Eaton Ave||Camden|ME|048431752|2300.00|2007-09-30|20 +|Buck|Barbara||1780 NE 138th St||North Miami|FL|331811316|50.00|2007-09-13|20 +|Buck|Barbara||1780 NE 138th St||North Miami|FL|331811316|50.00|2007-07-19|20 +|Buchman|Mark M||2530 Lawton Ave||San Luis Obispo|CA|934015622|460.80|2007-07-18|20 +|Bucher|Ida|M|1400 Warnall Ave||Los Angeles|CA|900245333|100.00|2007-07-10|20 +|Buchanek|Elizabeth||7917 Kentbury Dr||Bethesda|MD|208144615|50.00|2007-09-30|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|-500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|700.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|-700.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|1000.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|1300.00|2007-08-09|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|200.00|2007-08-14|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-07-25|20 +|Buchanan|John||4635 49th St NW||Washington|DC|200164320|200.09|2007-09-23|20 +|Harrison|Ryan||2247 3rd St||La Verne|CA|917504918|25.00|2007-07-26|20 +|BYNUM|HERBERT||332 SUNNYSIDE ROAD||TAMPA|FL|336177249|-500.00|2008-03-10|22 +|BYINGTON|MARGARET|E.|2633 MIDDLEBORO LANE N.E.||GRAND RAPIDS|MI|495061254|-2300.00|2008-03-03|22 +|BYERS|BOB|A.|13170 TELFAIR AVENUE||SYLMAR|CA|913423573|-2300.00|2008-03-07|22 +|BYERS|AUDREY||2658 LADBROOK WAY||THOUSAND OAKS|CA|913615073|-200.00|2008-03-07|22 +|BUSH|KRYSTIE||P.O. BOX 61046||DENVER|CO|802061046|-2300.00|2008-03-06|22 +|BUSH|ERIC||P.O. BOX 61046||DENVER|CO|802061046|-2300.00|2008-03-06|22 +|BURTON|SUSAN||9338 DEER CREEK DRIVE||TAMPA|FL|336472286|-2300.00|2008-03-05|22 +|BURTON|STEVEN|G.|9938 DEER CREEK DRIVE||TAMPA|FL|33647|-2300.00|2008-03-05|22 +|BURTON|GLENN|M.|4404 CHARLESTON COURT||TAMPA|FL|336092620|-2300.00|2008-03-05|22 +|BURKHARDT|CRAIG|S.|910 15TH STREET N.W.||WASHINGTON|DC|200052503|-500.00|2008-03-07|22 +|BURKHARDT|CRAIG|S.|910 15TH STREET N.W.||WASHINGTON|DC|200052503|-1000.00|2008-03-07|22 +|BURKHARDT|BARBARA||910 15TH STREET N.W.||WASHINGTON|DC|200052503|-500.00|2008-03-07|22 +|BURKE|SUZANNE|M.|3401 EVANSTON||SEATTLE|WA|981038677|-700.00|2008-03-05|22 +|BURKE|GAIL||165 E. 32ND STREET|APARTMENT 9E|NEW YORK|NY|100166014|-2000.00|2008-03-05|22 +|BURKE|DONALD|J.|12 LOMPOC||RANCHO SANTA MARGA|CA|926881817|-2300.00|2008-03-11|22 +|BURGERT|RONALD|L.|5723 PLUMTREE DRIVE||DALLAS|TX|752524926|-1000.00|2008-03-05|22 +|BULL|BARTLE|B.|439 E. 51ST STREET||NEW YORK|NY|100226473|-800.00|2008-03-10|22 +|BULL|BARTLE|B.|439 E. 51ST STREET||NEW YORK|NY|100226473|-1000.00|2008-03-10|22 +|BUKOWSKI|DANIEL|J.|702 S. WRIGHT STREET||NAPERVILLE|IL|605406736|-100.00|2008-03-10|22 +|BUISSON|MARGARET|A.|P.O. BOX 197029||LOUISVILLE|KY|402597029|-200.00|2008-03-11|22 +|BUCKLEY|WALTER|W.|1635 COUNTRY ROAD||BETHLEHEM|PA|180155718|-100.00|2008-03-05|22 +|BUCKLEY|MARJORIE|B.|1635 COUNTRY ROAD||BETHLEHEM|PA|180155718|-100.00|2008-03-05|22 +|BRUNO|JOHN||10136 WINDERMERE CHASE BLVD.||GOTHA|FL|347344707|-2300.00|2008-03-06|22 +|BRUNO|IRENE||10136 WINDERMERE CHASE BLVD.||GOTHA|FL|347344707|-2300.00|2008-03-06|22 +|BROWN|TIMOTHY|J.|26826 MARLOWE COURT||STEVENSON RANCH|CA|913811020|-2300.00|2008-03-06|22 +|Schuff|Bryan||1700 W Sweden Rd||Brockport|NY|14420|-25.00|2008-08-22|32 +|Hobbs|James||229 Cherry Lane||White House|TN|37188|-25.00|2008-08-19|32 +|Ranganath|Anoop||2507 Willard Drive||Charlottesville|VA|22903|-100.00|2008-04-21|32 +|Nystrom|Michael|A|93A Fairmont Street||Arlington|MA|02474|-503.00|2008-04-21|32 +|Muse|Nina|Jo|2915 Toro Canyon Rd||Austin|TX|78746|-50.00|2008-04-21|32 +|Waddell|James|L.|1823 Spel Lane SW||Rochester|MN|55902|-28.00|2008-04-21|32 +|Brucks|William|C.|PO Box 391||Corona del Mar|CA|92625|-150.00|2008-04-21|32 +|Kuehn|David||14502 West 93rd Street||Lenexa|KS|66215|-330.00|2008-04-21|32 +|Verster|Jeanette|M.|7220 SW 61st St||Miami|FL|331431807|-1000.00|2008-04-21|32 +|Uihlein|Richard||1396 N Waukegan Rd||Lake Forest|IL|600451147|-2300.00|2008-04-21|32 +|Eskenberry|Robert|P|10960 Gray Cir||Westminster|CO|80020|-223.00|2008-04-21|32 +|Froehling|Alan|L.|302 Broadway St||Mount Vernon|IL|628645116|-844.80|2008-04-21|32 +|Duryea|Marcia|A.|123 Bayview Ave||Amityville|NY|11701|-299.50|2008-04-21|32 +|Perreault|Louise||503 Brockridge Hunt Drive||Hampton|VA|23666|-34.08|2008-04-21|32 +|Rozenfeld|Timur||57 Herbert Road||Robbinsville|NJ|08691|-777.95|2008-04-21|32 +|Kazor|Christopher|M|707 Spindletree ave||Naperville|IL|60565|-2592.00|2008-04-21|32 +|Lehner|Thomas|S.|2701 Star Lane||Wadsworth|OH|44281|-200.00|2008-04-21|32 +|Plummer|Joseph||587 Blake Hill Rd||New Hampton|NH|032564424|-24.60|2008-04-21|32 +|Raught|Philip|M|4714 Plum Way||Pittsburgh|PA|15201|-1046.00|2008-04-21|32 +|Ferrara|Judith|D|1508 Waterford Road||Yardley|PA|19067|-1100.00|2008-04-21|32 +|Johnson|Cathleen|E.|1003 Justin Ln Apt 2016||Austin|TX|787572648|-14.76|2008-04-21|32 +|Sanford|Bradley||940 Post St #43||San Francisco|CA|94109|-24.53|2008-04-21|32 +|Gaarder|Bruce||PO Box 4085||Mountain Home AFB|ID|83648|-261.00|2008-04-21|32 +|Choe|Hyeokchan||207 Bridle Way||Fort Lee|NJ|070246302|-39.50|2008-04-21|32 +|Jacobs|Richard|G.|14337 Tawya Rd||Apple Valley|CA|923075545|-1000.00|2008-04-21|32 +|Aaronson|Rebecca||2000 Village Green Dr Apt 12||Mill Creek|WA|980125787|100.00|2008-02-08|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|25.00|2008-02-26|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|70.00|2008-02-25|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|100.00|2008-02-08|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|50.00|2008-02-29|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-29|34 +|Aaronson|Rebecca||2000 Village Green Dr Apt 12||Mill Creek|WA|980125787|100.00|2008-02-14|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-24|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-22|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-17|34 +|Reid|Elizabeth||73 W Patent Rd|OPHIR FARM NORTH|Bedford Hills|NY|105072222|-350.00|2008-08-28|34 +|Reich|Thomas||499 Park Ave||New York|NY|100221240|-2300.00|2008-08-28|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-08|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-03|34 +|Aaron|Sharron||1804 E Montgomery St||Broken Arrow|OK|740121840|500.00|2008-02-09|34 +|Aaron|Patricia||418 NW 35th St||Oklahoma City|OK|731188602|200.00|2008-02-26|34 +|Aaron|Patricia||418 NW 35th St||Oklahoma City|OK|731188602|100.00|2008-02-12|34 +|Aaron|Jim||2178 Fairway Cir||Canton|MI|481885097|300.00|2008-02-07|34 +|Aaron|Jim||2178 Fairway Cir||Canton|MI|481885097|200.00|2008-02-29|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|70.00|2008-02-29|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|50.00|2008-02-07|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|100.00|2008-02-03|34 +|Aaron|Barbara||2298 Pacific Ave # 6||San Francisco|CA|941151435|1000.00|2008-02-11|34 +|Aanonsen|Lin||897 Raymond Ave||Saint Paul|MN|551141508|250.00|2008-02-21|34 +|Aanonsen|Lin||897 Raymond Ave||Saint Paul|MN|551141508|100.00|2008-02-08|34 +|BOURNE|TRAVIS||LAGE KAART 77||BRASSCHATT||02930|-500.00|2008-11-20|35 +|SECRIST|BRIAN|L.|3 MULE DEER TRAIL||LITTLETON|CO|801275722|-1000.00|2008-04-07|35 +|TOLLESTRUP|TRAVIS|W.|16331 WINECREEK RD.||SAN DIEGO|CA|92127|-1000.00|2008-05-15|35 +|ACCORD|DEAN|C.|8813 ROBINSON RIDGE ROAD||LAS VEGAS|NV|891175812|500.00|2007-07-13|35 +|ABTS|HENRY||P. O. BOX 7299||INCLINE VILLAGE|NV|894527299|100.00|2007-07-13|35 +|ABSHIER|LANNY||14191 S.E. HIGHWAY 301||SUMMERFIELD|FL|34491|500.00|2007-09-25|35 +|ABSHIER|DIANA||14191 S.E. HIGHWAY 301||SUMMERFIELD|FL|34491|500.00|2007-09-25|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|50.00|2007-09-30|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|150.00|2007-08-09|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|50.00|2007-07-19|35 +|ABRAMOWITZ|NIRA||411 HARBOR ROAD||SOUTHPORT|CT|068901376|2300.00|2007-09-14|35 +|ABRAMS|MICHAEL||7910 WOODMONT AVENUE||BETHESDA|MD|208143002|250.00|2007-09-29|35 +|ABRAMOWITZ|KEN||200 CENTRAL PARK S.|APARTMENT 31A|NEW YORK|NY|100191448|300.00|2007-09-11|35 +|ABOUBAKARE|NASAR||1400 SAN MIGUEL DRIVE||CORONA DEL MAR|CA|926251300|1000.00|2007-07-09|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-09-25|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-09-17|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-08-31|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-08-14|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-08-06|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-07-10|35 +|ABDELLA|THOMAS|M.|4231 MONUMENT WALL WAY #340||FAIRFAX|VA|220308440|50.00|2007-09-30|35 +|ABBOTT|WELDON|S.|777 EAST SOUTH TEMPLE 4E||SALT LAKE CITY|UT|841021269|100.00|2007-09-29|35 +|ABBOTT|WELDON|S.|777 EAST SOUTH TEMPLE 4E||SALT LAKE CITY|UT|841021269|50.00|2007-08-09|35 +|ABBOTT|GERALD|F.|389 BENEFIT STREET||PROVIDENCE|RI|029032946|100.00|2007-09-15|35 +|ABBOTT|GERALD|F.|389 BENEFIT STREET||PROVIDENCE|RI|029032946|100.00|2007-08-15|35 +|ABEDIN|ZAINUL||715 N. CENTRAL AVENUE|SUITE 212|GLENDALE|CA|912031164|500.00|2008-01-21|37 +|ABBOTT|SYBIL|F.|446 GAMES DRIVE||RENO|NV|895093326|75.00|2008-01-08|37 +|ABBOTT|SYBIL|F.|446 GAMES DRIVE||RENO|NV|895093326|50.00|2008-01-08|37 +|ABBOTT|RONALD|LEANDER|5453 HAWTHORNE STREET||MONTCLAIR|CA|917632551|200.00|2008-01-31|37 +|ABBOTT|RONALD|LEANDER|5453 HAWTHORNE STREET||MONTCLAIR|CA|917632551|100.00|2008-01-08|37 +|ABBOTT|ROBERT|A.|3061 LOREE ROAD||DECKERVILLE|MI|484279763|500.00|2008-01-21|37 +|ABBOTT|MIKE|E.|4516 OSPREY LNDG||NICEVILLE|FL|325786810|1000.00|2008-01-15|37 +|ABBOT|DAVID|M.|56 SALEM STREET||ANDOVER|MA|018102114|200.00|2008-01-21|37 +|ABBO|PAULINE|MORENCY|10720 JACOB LANE||WHITE LAKE|MI|483862274|35.00|2008-01-07|37 +|ABATE|MARIA|ELENA|1291 NIGHTINGALE AVENUE||MIAMI SPRINGS|FL|331663832|2600.00|2008-01-25|37 +|ABAIR|PETER||40 EVANS STREET||WATERTOWN|MA|024722150|25.00|2008-01-09|37 +|ABACHERLI|SHIRLEY|M.|29875 NEWPORT ROAD||MENIFEE|CA|925849524|150.00|2008-01-28|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|300.00|2008-01-30|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|410.00|2008-01-15|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|500.00|2008-01-09|37 +|ABEL|JOHN|H.|422 THOMAS STREET||BETHLEHEM|PA|180153316|200.00|2008-01-22|37 +|ABEL|MARLING|L.|14 HANGING MOSS LANE||GREENVILLE|SC|296155069|100.00|2008-01-22|37 +|ABEL|RUDOLPH||4532 OCEAN BLVD.|# 108|SARASOTA|FL|342421337|100.00|2008-01-08|37 +|ABELE|RODNEY||3620 METAIRIE HEIGHTS AVENUE||METAIRIE|LA|700021823|500.00|2008-01-15|37 +|ABERCROMBIE|DENIS||11811 WATER OAK CT||MAGNOLIA|TX|773546270|500.00|2008-01-30|37 +|ABESHAUS|MERRILL|M.|1801 N. HEREFORD DRIVE||FLAGSTAFF|AZ|860011121|120.00|2008-01-16|37 +|ABRAHAM|GEORGE||P.O. BOX 1504||LAKE CHARLES|LA|706021504|800.00|2008-01-17|37 +|ABRAHAMSON|PETER|J.|1030 W. ROSCOE STREET||CHICAGO|IL|606572207|50.00|2008-01-25|37 +|ABRAHAM|SALEM|A.|P.O. BOX 7||CANADIAN|TX|790140007|1000.00|2008-01-17|37 +|ABRAHAM|SALEM|A.|P.O. BOX 7||CANADIAN|TX|790140007|1300.00|2008-01-30|37 diff --git a/lecture21/presidential/presidential.py b/lecture21/presidential/presidential.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# vim: foldmethod=marker +# File : presidential.py +# Description: SQLite database class demo for presidential candidates in 2008 +# Copyright 2022 Harvard University. All Rights Reserved. +import sqlite3 + + +# main() {{{1 +def main(): + # create a connection to the database {{{2 + # https://www.python.org/dev/peps/pep-0249/#connection-objects + db = sqlite3.connect('presidential.sqlite') + + # obtain a cursor object for the `db` database {{{2 + # https://www.python.org/dev/peps/pep-0249/#cursor-objects + cursor = db.cursor() + + # drop existing tables to start fresh for the purpose of this demo + cursor.execute('DROP TABLE IF EXISTS candidates') + cursor.execute('DROP TABLE IF EXISTS contributors') + + # turn on FOREIGN KEY support in SQLite (off by default). If + # foreign_keys=1, a sqlite3.IntegrityError will be raised if a FOREIGN KEY + # constraint fails. + cursor.execute('PRAGMA foreign_keys=1') + + # create a table for the `candidates` {{{2 + cursor.execute( + '''CREATE TABLE candidates ( + id INTEGER PRIMARY KEY NOT NULL, + first_name TEXT, + last_name TEXT, + middle_initial TEXT, + party TEXT NOT NULL)''' + ) + db.commit() # commit the changes to the database + + # insert rows {{{2 + cursor.execute( + '''INSERT INTO candidates + (id, first_name, last_name, middle_initial, party) + VALUES (?, ?, ?, ?, ?)''', (16, 'Mike', 'Huckabee', '', 'R') + ) + + cursor.execute( + '''INSERT INTO candidates + (id, first_name, last_name, middle_initial, party) + VALUES (?, ?, ?, ?, ?)''', (32, 'Ron', 'Paul', '', 'R') + ) + + cursor.execute( + '''INSERT INTO candidates + (id, first_name, last_name, middle_initial, party) + VALUES (?, ?, ?, ?, ?)''', (20, 'Barack', 'Obama', '', 'D') + ) + + db.commit() # commit the changes to the database + + # queries {{{2 + # getting all columns returned in rows {{{3 + cursor.execute("SELECT * FROM candidates") + rows = cursor.fetchall() + print(f'All rows and columns: got {len(rows)} rows') + for row in rows: + print(row) + + # explicit selects with WHERE {{{3 + cursor.execute("SELECT * FROM candidates WHERE first_name = 'mike'") + # cursor.execute("SELECT * FROM candidates WHERE LOWER(first_name) = 'mike'") + rows = cursor.fetchall() + print(f"Looking for 'mike': got {len(rows)} rows") + for row in rows: + print(row) + + # explicit select of specified column {{{3 + cursor.execute("SELECT first_name FROM candidates") + rows = cursor.fetchall() + print(f"Looking for first_name: got {len(rows)} rows") + for row in rows: + print(row) + + # create a table for the `contributors` {{{2 + cursor.execute( + '''CREATE TABLE contributors ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + last_name TEXT, + first_name TEXT, + middle_name TEXT, + street_1 TEXT, + street_2 TEXT, + city TEXT, + state TEXT, + zip TEXT, + amount FLOAT(7,3), + date DATETIME, + candidate_id INTEGER NOT NULL, + FOREIGN KEY(candidate_id) REFERENCES candidates(id))''' + ) + db.commit() # commit the changes to the database + + # add contributors {{{2 + contributors = [ + ( + "Agee", "Steven", "", "549 Laurel Branch Road", "", "Floyd", "VA", + int(24091), 500.0, '2007-06-30', 16 + ), + ( + "Buck", "Jay", "K.", "1855 Old Willow Rd Unit 322", "", + "Northfield", "IL", int(600932918), 200.0, '2007-09-12', 20 + ), + ( + "Choe", "Hyeokchan", "", "207 Bridle Way", "", "Fort Lee", "NJ", + int(70246302), -39.50, '2008-04-21', 32 + ), + ] + + cursor.executemany( + '''INSERT INTO contributors + (last_name, first_name, middle_name, street_1, street_2, city, state, zip, + amount, date, candidate_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', + contributors + ) + db.commit() + + cursor.execute('SELECT last_name FROM contributors WHERE amount <= 200') + for row in cursor.fetchall(): + print(row) + + # foreign key violation {{{2 + try: + cursor.execute( + '''INSERT INTO contributors + (last_name, first_name, middle_name, street_1, street_2, city, state, zip, + amount, date, candidate_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', + ( + "Buckler", "Steve", "", "24351 Armada Dr.", "", "Dana Point", + "CA", int(926291), 50, '2007-07-30', 34 + ) + ) + except sqlite3.IntegrityError as e: + # we expect this error and allow to pass such that we can close the + # database in the next section + print(e) + pass + + # close database when done {{{2 + db.close() + + +# __main__ {{{1 +if __name__ == "__main__": + main() diff --git a/lecture22/candidates.txt b/lecture22/candidates.txt @@ -0,0 +1,18 @@ +id|first_name|last_name|middle_name|party +33|Joseph|Biden||D +36|Samuel|Brownback||R +34|Hillary|Clinton|R.|D +39|Christopher|Dodd|J.|D +26|John|Edwards||D +22|Rudolph|Giuliani||R +24|Mike|Gravel||D +16|Mike|Huckabee||R +30|Duncan|Hunter||R +31|Dennis|Kucinich||D +37|John|McCain||R +20|Barack|Obama||D +32|Ron|Paul||R +29|Bill|Richardson||D +35|Mitt|Romney||R +38|Tom|Tancredo||R +41|Fred|Thompson|D.|R diff --git a/lecture22/contributors.txt b/lecture22/contributors.txt @@ -0,0 +1,176 @@ +id|last_name|first_name|middle_name|street_1|street_2|city|state|zip|amount|date|candidate_id +|Agee|Steven||549 Laurel Branch Road||Floyd|VA|24091|500.00|2007-06-30|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|250.00|2007-05-16|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|50.00|2007-06-18|16 +|Ahrens|Don||4034 Rennellwood Way||Pleasanton|CA|94566|100.00|2007-06-21|16 +|Akin|Charles||10187 Sugar Creek Road||Bentonville|AR|72712|100.00|2007-06-16|16 +|Akin|Mike||181 Baywood Lane||Monticello|AR|71655|1500.00|2007-05-18|16 +|Akin|Rebecca||181 Baywood Lane||Monticello|AR|71655|500.00|2007-05-18|16 +|Aldridge|Brittni||808 Capitol Square Place, SW||Washington|DC|20024|250.00|2007-06-06|16 +|Allen|John D.||1052 Cannon Mill Drive||North Augusta|SC|29860|1000.00|2007-06-11|16 +|Allen|John D.||1052 Cannon Mill Drive||North Augusta|SC|29860|1300.00|2007-06-29|16 +|Allison|John W.||P.O. Box 1089||Conway|AR|72033|1000.00|2007-05-18|16 +|Allison|Rebecca||3206 Summit Court||Little Rock|AR|72227|1000.00|2007-04-25|16 +|Allison|Rebecca||3206 Summit Court||Little Rock|AR|72227|200.00|2007-06-12|16 +|Altes|R.D.||8600 Moody Road||Fort Smith|AR|72903|2300.00|2007-06-21|16 +|Andres|Dale||1160 Glen Oaks Drive||West Des Moines|IA|50266|250.00|2007-06-06|16 +|Anthony|John||211 Long Island Drive||Hot Springs|AR|71913|2300.00|2007-06-12|16 +|Arbogast|Robert||12900 State Route 56 SE||Mount Sterling|OH|43143|500.00|2007-04-08|16 +|Arbogast|Robert||12900 State Route 56 SE||Mount Sterling|OH|43143|100.00|2007-06-22|16 +|Ardle|William||412 Dakota Avenue||Springfield|OH|45504|50.00|2007-06-28|16 +|Atiq|Omar||7200 S Hazel Street||Pine Bluff|AR|71603|1000.00|2007-05-18|16 +|Atiq|Omar||7200 S Hazel Street||Pine Bluff|AR|71603|1000.00|2007-06-27|16 +|Baker|David||2550 Adamsbrooke Drive||Conway|AR|72034|2300.00|2007-04-11|16 +|Bancroft|David||2934 Broderick Street||San Francisco|CA|94123|250.00|2007-04-24|16 +|Banks|Charles||P.O. Box 251310||Little Rock|AR|72225|1000.00|2007-05-14|16 +|Barbee|John||516 Kellyridge Drive||Apex|NC|27502|500.00|2007-05-23|16 +|Buckler|Steve||24351 Armada Dr||Dana Point|CA|926291306|50.00|2007-07-30|20 +|Buckler|Steve||24351 Armada Dr||Dana Point|CA|926291306|25.00|2007-08-16|20 +|Buckheit|Bruce||8904 KAREN DR||FAIRFAX|VA|220312731|100.00|2007-09-19|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|2300.00|2007-08-14|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|-2300.00|2007-08-14|20 +|Buckel|Linda||PO Box 683130||Park City|UT|840683130|4600.00|2007-08-14|20 +|Buck|Thomas||4206 Terrace Street||Kansas City|MO|64111|100.00|2007-09-25|20 +|Buck|Jay|K.|1855 Old Willow Rd Unit 322||Northfield|IL|600932918|200.00|2007-09-12|20 +|Buck|Blaine|M|45 Eaton Ave||Camden|ME|048431752|2300.00|2007-09-30|20 +|Buck|Barbara||1780 NE 138th St||North Miami|FL|331811316|50.00|2007-09-13|20 +|Buck|Barbara||1780 NE 138th St||North Miami|FL|331811316|50.00|2007-07-19|20 +|Buchman|Mark M||2530 Lawton Ave||San Luis Obispo|CA|934015622|460.80|2007-07-18|20 +|Bucher|Ida|M|1400 Warnall Ave||Los Angeles|CA|900245333|100.00|2007-07-10|20 +|Buchanek|Elizabeth||7917 Kentbury Dr||Bethesda|MD|208144615|50.00|2007-09-30|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|-500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-09-24|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|700.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|-700.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|1000.00|2007-08-28|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|1300.00|2007-08-09|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|200.00|2007-08-14|20 +|Buchanan|John||2025 NW 29th Rd||Boca Raton|FL|334316303|500.00|2007-07-25|20 +|Buchanan|John||4635 49th St NW||Washington|DC|200164320|200.09|2007-09-23|20 +|Harrison|Ryan||2247 3rd St||La Verne|CA|917504918|25.00|2007-07-26|20 +|BYNUM|HERBERT||332 SUNNYSIDE ROAD||TAMPA|FL|336177249|-500.00|2008-03-10|22 +|BYINGTON|MARGARET|E.|2633 MIDDLEBORO LANE N.E.||GRAND RAPIDS|MI|495061254|-2300.00|2008-03-03|22 +|BYERS|BOB|A.|13170 TELFAIR AVENUE||SYLMAR|CA|913423573|-2300.00|2008-03-07|22 +|BYERS|AUDREY||2658 LADBROOK WAY||THOUSAND OAKS|CA|913615073|-200.00|2008-03-07|22 +|BUSH|KRYSTIE||P.O. BOX 61046||DENVER|CO|802061046|-2300.00|2008-03-06|22 +|BUSH|ERIC||P.O. BOX 61046||DENVER|CO|802061046|-2300.00|2008-03-06|22 +|BURTON|SUSAN||9338 DEER CREEK DRIVE||TAMPA|FL|336472286|-2300.00|2008-03-05|22 +|BURTON|STEVEN|G.|9938 DEER CREEK DRIVE||TAMPA|FL|33647|-2300.00|2008-03-05|22 +|BURTON|GLENN|M.|4404 CHARLESTON COURT||TAMPA|FL|336092620|-2300.00|2008-03-05|22 +|BURKHARDT|CRAIG|S.|910 15TH STREET N.W.||WASHINGTON|DC|200052503|-500.00|2008-03-07|22 +|BURKHARDT|CRAIG|S.|910 15TH STREET N.W.||WASHINGTON|DC|200052503|-1000.00|2008-03-07|22 +|BURKHARDT|BARBARA||910 15TH STREET N.W.||WASHINGTON|DC|200052503|-500.00|2008-03-07|22 +|BURKE|SUZANNE|M.|3401 EVANSTON||SEATTLE|WA|981038677|-700.00|2008-03-05|22 +|BURKE|GAIL||165 E. 32ND STREET|APARTMENT 9E|NEW YORK|NY|100166014|-2000.00|2008-03-05|22 +|BURKE|DONALD|J.|12 LOMPOC||RANCHO SANTA MARGA|CA|926881817|-2300.00|2008-03-11|22 +|BURGERT|RONALD|L.|5723 PLUMTREE DRIVE||DALLAS|TX|752524926|-1000.00|2008-03-05|22 +|BULL|BARTLE|B.|439 E. 51ST STREET||NEW YORK|NY|100226473|-800.00|2008-03-10|22 +|BULL|BARTLE|B.|439 E. 51ST STREET||NEW YORK|NY|100226473|-1000.00|2008-03-10|22 +|BUKOWSKI|DANIEL|J.|702 S. WRIGHT STREET||NAPERVILLE|IL|605406736|-100.00|2008-03-10|22 +|BUISSON|MARGARET|A.|P.O. BOX 197029||LOUISVILLE|KY|402597029|-200.00|2008-03-11|22 +|BUCKLEY|WALTER|W.|1635 COUNTRY ROAD||BETHLEHEM|PA|180155718|-100.00|2008-03-05|22 +|BUCKLEY|MARJORIE|B.|1635 COUNTRY ROAD||BETHLEHEM|PA|180155718|-100.00|2008-03-05|22 +|BRUNO|JOHN||10136 WINDERMERE CHASE BLVD.||GOTHA|FL|347344707|-2300.00|2008-03-06|22 +|BRUNO|IRENE||10136 WINDERMERE CHASE BLVD.||GOTHA|FL|347344707|-2300.00|2008-03-06|22 +|BROWN|TIMOTHY|J.|26826 MARLOWE COURT||STEVENSON RANCH|CA|913811020|-2300.00|2008-03-06|22 +|Schuff|Bryan||1700 W Sweden Rd||Brockport|NY|14420|-25.00|2008-08-22|32 +|Hobbs|James||229 Cherry Lane||White House|TN|37188|-25.00|2008-08-19|32 +|Ranganath|Anoop||2507 Willard Drive||Charlottesville|VA|22903|-100.00|2008-04-21|32 +|Nystrom|Michael|A|93A Fairmont Street||Arlington|MA|02474|-503.00|2008-04-21|32 +|Muse|Nina|Jo|2915 Toro Canyon Rd||Austin|TX|78746|-50.00|2008-04-21|32 +|Waddell|James|L.|1823 Spel Lane SW||Rochester|MN|55902|-28.00|2008-04-21|32 +|Brucks|William|C.|PO Box 391||Corona del Mar|CA|92625|-150.00|2008-04-21|32 +|Kuehn|David||14502 West 93rd Street||Lenexa|KS|66215|-330.00|2008-04-21|32 +|Verster|Jeanette|M.|7220 SW 61st St||Miami|FL|331431807|-1000.00|2008-04-21|32 +|Uihlein|Richard||1396 N Waukegan Rd||Lake Forest|IL|600451147|-2300.00|2008-04-21|32 +|Eskenberry|Robert|P|10960 Gray Cir||Westminster|CO|80020|-223.00|2008-04-21|32 +|Froehling|Alan|L.|302 Broadway St||Mount Vernon|IL|628645116|-844.80|2008-04-21|32 +|Duryea|Marcia|A.|123 Bayview Ave||Amityville|NY|11701|-299.50|2008-04-21|32 +|Perreault|Louise||503 Brockridge Hunt Drive||Hampton|VA|23666|-34.08|2008-04-21|32 +|Rozenfeld|Timur||57 Herbert Road||Robbinsville|NJ|08691|-777.95|2008-04-21|32 +|Kazor|Christopher|M|707 Spindletree ave||Naperville|IL|60565|-2592.00|2008-04-21|32 +|Lehner|Thomas|S.|2701 Star Lane||Wadsworth|OH|44281|-200.00|2008-04-21|32 +|Plummer|Joseph||587 Blake Hill Rd||New Hampton|NH|032564424|-24.60|2008-04-21|32 +|Raught|Philip|M|4714 Plum Way||Pittsburgh|PA|15201|-1046.00|2008-04-21|32 +|Ferrara|Judith|D|1508 Waterford Road||Yardley|PA|19067|-1100.00|2008-04-21|32 +|Johnson|Cathleen|E.|1003 Justin Ln Apt 2016||Austin|TX|787572648|-14.76|2008-04-21|32 +|Sanford|Bradley||940 Post St #43||San Francisco|CA|94109|-24.53|2008-04-21|32 +|Gaarder|Bruce||PO Box 4085||Mountain Home AFB|ID|83648|-261.00|2008-04-21|32 +|Choe|Hyeokchan||207 Bridle Way||Fort Lee|NJ|070246302|-39.50|2008-04-21|32 +|Jacobs|Richard|G.|14337 Tawya Rd||Apple Valley|CA|923075545|-1000.00|2008-04-21|32 +|Aaronson|Rebecca||2000 Village Green Dr Apt 12||Mill Creek|WA|980125787|100.00|2008-02-08|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|25.00|2008-02-26|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|70.00|2008-02-25|34 +|Aarons|Elaine||481 Buck Island Rd Apt 17A|APT 17A|West Yarmouth|MA|026733300|100.00|2008-02-08|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|50.00|2008-02-29|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-29|34 +|Aaronson|Rebecca||2000 Village Green Dr Apt 12||Mill Creek|WA|980125787|100.00|2008-02-14|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-24|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-22|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-17|34 +|Reid|Elizabeth||73 W Patent Rd|OPHIR FARM NORTH|Bedford Hills|NY|105072222|-350.00|2008-08-28|34 +|Reich|Thomas||499 Park Ave||New York|NY|100221240|-2300.00|2008-08-28|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-08|34 +|Aaron|Shirley||101 Cherry Ave||Havana|FL|323331311|100.00|2008-02-03|34 +|Aaron|Sharron||1804 E Montgomery St||Broken Arrow|OK|740121840|500.00|2008-02-09|34 +|Aaron|Patricia||418 NW 35th St||Oklahoma City|OK|731188602|200.00|2008-02-26|34 +|Aaron|Patricia||418 NW 35th St||Oklahoma City|OK|731188602|100.00|2008-02-12|34 +|Aaron|Jim||2178 Fairway Cir||Canton|MI|481885097|300.00|2008-02-07|34 +|Aaron|Jim||2178 Fairway Cir||Canton|MI|481885097|200.00|2008-02-29|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|70.00|2008-02-29|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|50.00|2008-02-07|34 +|Aaron|Carole||PO Box 1806||Ogunquit|ME|039071806|100.00|2008-02-03|34 +|Aaron|Barbara||2298 Pacific Ave # 6||San Francisco|CA|941151435|1000.00|2008-02-11|34 +|Aanonsen|Lin||897 Raymond Ave||Saint Paul|MN|551141508|250.00|2008-02-21|34 +|Aanonsen|Lin||897 Raymond Ave||Saint Paul|MN|551141508|100.00|2008-02-08|34 +|BOURNE|TRAVIS||LAGE KAART 77||BRASSCHATT||02930|-500.00|2008-11-20|35 +|SECRIST|BRIAN|L.|3 MULE DEER TRAIL||LITTLETON|CO|801275722|-1000.00|2008-04-07|35 +|TOLLESTRUP|TRAVIS|W.|16331 WINECREEK RD.||SAN DIEGO|CA|92127|-1000.00|2008-05-15|35 +|ACCORD|DEAN|C.|8813 ROBINSON RIDGE ROAD||LAS VEGAS|NV|891175812|500.00|2007-07-13|35 +|ABTS|HENRY||P. O. BOX 7299||INCLINE VILLAGE|NV|894527299|100.00|2007-07-13|35 +|ABSHIER|LANNY||14191 S.E. HIGHWAY 301||SUMMERFIELD|FL|34491|500.00|2007-09-25|35 +|ABSHIER|DIANA||14191 S.E. HIGHWAY 301||SUMMERFIELD|FL|34491|500.00|2007-09-25|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|50.00|2007-09-30|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|150.00|2007-08-09|35 +|ABREU|KEVIN|M.|1305 GARDEN GLEN LANE||PEARLAND|TX|775816547|50.00|2007-07-19|35 +|ABRAMOWITZ|NIRA||411 HARBOR ROAD||SOUTHPORT|CT|068901376|2300.00|2007-09-14|35 +|ABRAMS|MICHAEL||7910 WOODMONT AVENUE||BETHESDA|MD|208143002|250.00|2007-09-29|35 +|ABRAMOWITZ|KEN||200 CENTRAL PARK S.|APARTMENT 31A|NEW YORK|NY|100191448|300.00|2007-09-11|35 +|ABOUBAKARE|NASAR||1400 SAN MIGUEL DRIVE||CORONA DEL MAR|CA|926251300|1000.00|2007-07-09|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-09-25|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-09-17|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-08-31|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|75.00|2007-08-14|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-08-06|35 +|ABEGG|PATRICIA|T.|1862 E. 5150 S.||SALT LAKE CITY|UT|841176911|25.00|2007-07-10|35 +|ABDELLA|THOMAS|M.|4231 MONUMENT WALL WAY #340||FAIRFAX|VA|220308440|50.00|2007-09-30|35 +|ABBOTT|WELDON|S.|777 EAST SOUTH TEMPLE 4E||SALT LAKE CITY|UT|841021269|100.00|2007-09-29|35 +|ABBOTT|WELDON|S.|777 EAST SOUTH TEMPLE 4E||SALT LAKE CITY|UT|841021269|50.00|2007-08-09|35 +|ABBOTT|GERALD|F.|389 BENEFIT STREET||PROVIDENCE|RI|029032946|100.00|2007-09-15|35 +|ABBOTT|GERALD|F.|389 BENEFIT STREET||PROVIDENCE|RI|029032946|100.00|2007-08-15|35 +|ABEDIN|ZAINUL||715 N. CENTRAL AVENUE|SUITE 212|GLENDALE|CA|912031164|500.00|2008-01-21|37 +|ABBOTT|SYBIL|F.|446 GAMES DRIVE||RENO|NV|895093326|75.00|2008-01-08|37 +|ABBOTT|SYBIL|F.|446 GAMES DRIVE||RENO|NV|895093326|50.00|2008-01-08|37 +|ABBOTT|RONALD|LEANDER|5453 HAWTHORNE STREET||MONTCLAIR|CA|917632551|200.00|2008-01-31|37 +|ABBOTT|RONALD|LEANDER|5453 HAWTHORNE STREET||MONTCLAIR|CA|917632551|100.00|2008-01-08|37 +|ABBOTT|ROBERT|A.|3061 LOREE ROAD||DECKERVILLE|MI|484279763|500.00|2008-01-21|37 +|ABBOTT|MIKE|E.|4516 OSPREY LNDG||NICEVILLE|FL|325786810|1000.00|2008-01-15|37 +|ABBOT|DAVID|M.|56 SALEM STREET||ANDOVER|MA|018102114|200.00|2008-01-21|37 +|ABBO|PAULINE|MORENCY|10720 JACOB LANE||WHITE LAKE|MI|483862274|35.00|2008-01-07|37 +|ABATE|MARIA|ELENA|1291 NIGHTINGALE AVENUE||MIAMI SPRINGS|FL|331663832|2600.00|2008-01-25|37 +|ABAIR|PETER||40 EVANS STREET||WATERTOWN|MA|024722150|25.00|2008-01-09|37 +|ABACHERLI|SHIRLEY|M.|29875 NEWPORT ROAD||MENIFEE|CA|925849524|150.00|2008-01-28|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|300.00|2008-01-30|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|410.00|2008-01-15|37 +|AARONS|CHARLES||1730 SHORE DRIVE||ANCHORAGE|AK|995153207|500.00|2008-01-09|37 +|ABEL|JOHN|H.|422 THOMAS STREET||BETHLEHEM|PA|180153316|200.00|2008-01-22|37 +|ABEL|MARLING|L.|14 HANGING MOSS LANE||GREENVILLE|SC|296155069|100.00|2008-01-22|37 +|ABEL|RUDOLPH||4532 OCEAN BLVD.|# 108|SARASOTA|FL|342421337|100.00|2008-01-08|37 +|ABELE|RODNEY||3620 METAIRIE HEIGHTS AVENUE||METAIRIE|LA|700021823|500.00|2008-01-15|37 +|ABERCROMBIE|DENIS||11811 WATER OAK CT||MAGNOLIA|TX|773546270|500.00|2008-01-30|37 +|ABESHAUS|MERRILL|M.|1801 N. HEREFORD DRIVE||FLAGSTAFF|AZ|860011121|120.00|2008-01-16|37 +|ABRAHAM|GEORGE||P.O. BOX 1504||LAKE CHARLES|LA|706021504|800.00|2008-01-17|37 +|ABRAHAMSON|PETER|J.|1030 W. ROSCOE STREET||CHICAGO|IL|606572207|50.00|2008-01-25|37 +|ABRAHAM|SALEM|A.|P.O. BOX 7||CANADIAN|TX|790140007|1000.00|2008-01-17|37 +|ABRAHAM|SALEM|A.|P.O. BOX 7||CANADIAN|TX|790140007|1300.00|2008-01-30|37 diff --git a/lecture22/lecture22.ipynb b/lecture22/lecture22.ipynb @@ -0,0 +1,681 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "93cbae4c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Lecture 22\n", + "## Tuesday, November 15th, 2022\n", + "### SQL Database Exercise (I)" + ] + }, + { + "cell_type": "markdown", + "id": "c4895afc", + "metadata": {}, + "source": [ + "# `SQLite` Exercises\n", + "\n", + "Today you will work with the `candidates.txt` and `contributors.txt` datasets to\n", + "create a database in `python` using `SQLite`. This is a hands-on lecture.\n", + "You may do these tasks in a Jupyter notebook.\n", + "\n", + "The exercises will consist of a sequence of steps to help illustrate basic\n", + "commands." + ] + }, + { + "cell_type": "markdown", + "id": "813913ac", + "metadata": { + "lines_to_next_cell": 2, + "slideshow": { + "slide_type": "subslide" + } + }, + "source": [ + "<a id='deliverables'></a>\n", + "# Exercise Deliverables\n", + "\n", + "> 1. Copy the Jupyter notebook along with `candidates.txt` and\n", + "> `contributors.txt` into `lab/pp12` in your private Git repository and\n", + "> commit on your default branch.\n", + "> 2. For each step in the exercise notebook, there are instructions labeled\n", + "> \"**Do the following:**\" (except for Setup and Interlude). Put all the code\n", + "> for those instructions in _code cell(s) immediately following the\n", + "> instructions_. The code in those cells should be regular Python code.\n", + "> You should place comments where appropriate that describe your intentions.\n", + "> **Note:** To get the\n", + "> `pandas` tables to display in a cell, use `display()`.\n", + "> 3. Save and close your database. Be sure to upload your database in\n", + "> `lab/pp12` as well. Please name your database **`lecture22.sqlite`**." + ] + }, + { + "cell_type": "markdown", + "id": "35b6d1a9", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "[Setup](#setup)\n", + "\n", + "[Interlude](#interlude): Not required but highly recommended.\n", + "\n", + "[Step 1](#step_1)\n", + "\n", + "[Step 2](#step_2)\n", + "\n", + "[Step 3](#step_3)\n", + "\n", + "[Step 4](#step_4)\n", + "\n", + "[Step 5](#step_5)\n", + "\n", + "[Step 6](#step_6)\n", + "\n", + "[Step 7](#step_7)\n", + "\n", + "[Step 8](#step_8)\n" + ] + }, + { + "cell_type": "markdown", + "id": "3b6570e6", + "metadata": {}, + "source": [ + "<a id='setup'></a>\n", + "# Setup\n", + "\n", + "You should import `sqlite3` again like in the lecture." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5e575a5", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3" + ] + }, + { + "cell_type": "markdown", + "id": "b0aca704", + "metadata": {}, + "source": [ + "We will also use a basic `pandas` feature to display tables in the database." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "976d7b5f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "pd.set_option('display.width', 500)\n", + "pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', 100)\n", + "pd.set_option('display.notebook_repr_html', True)\n", + "\n", + "from IPython.display import display" + ] + }, + { + "cell_type": "markdown", + "id": "082acea0", + "metadata": {}, + "source": [ + "Now we create the tables in the database (similar to lecture)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4a0d354", + "metadata": {}, + "outputs": [], + "source": [ + "db = sqlite3.connect('lecture22.sqlite')\n", + "cursor = db.cursor()\n", + "cursor.execute(\"DROP TABLE IF EXISTS candidates\")\n", + "cursor.execute(\"DROP TABLE IF EXISTS contributors\")\n", + "cursor.execute(\"PRAGMA foreign_keys=1\")\n", + "\n", + "cursor.execute('''CREATE TABLE candidates (\n", + " id INTEGER PRIMARY KEY NOT NULL, \n", + " first_name TEXT, \n", + " last_name TEXT, \n", + " middle_name TEXT, \n", + " party TEXT NOT NULL)''')\n", + "\n", + "db.commit() # Commit changes to the database\n", + "\n", + "cursor.execute('''CREATE TABLE contributors (\n", + " id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, \n", + " last_name TEXT, \n", + " first_name TEXT, \n", + " middle_name TEXT, \n", + " street_1 TEXT, \n", + " street_2 TEXT, \n", + " city TEXT, \n", + " state TEXT, \n", + " zip TEXT, \n", + " amount REAL, \n", + " date DATETIME, \n", + " candidate_id INTEGER NOT NULL, \n", + " FOREIGN KEY(candidate_id) REFERENCES candidates(id))''')\n", + "\n", + "db.commit()" + ] + }, + { + "cell_type": "markdown", + "id": "fb30c3d5", + "metadata": {}, + "source": [ + "Next we load the data for the candidates:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b70dc4e", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "with open (\"candidates.txt\") as candidates:\n", + " next(candidates) # jump over the header\n", + " for line in candidates.readlines():\n", + " cid, first_name, last_name, middle_name, party = line.strip().split('|')\n", + " vals_to_insert = (int(cid), first_name, last_name, middle_name, party)\n", + " cursor.execute('''INSERT INTO candidates \n", + " (id, first_name, last_name, middle_name, party)\n", + " VALUES (?, ?, ?, ?, ?)''', vals_to_insert)\n", + "db.commit()" + ] + }, + { + "cell_type": "markdown", + "id": "ec092362", + "metadata": {}, + "source": [ + "<a id='interlude'></a>\n", + "## Interlude\n", + "\n", + "Now that you have values in the tables of the database, it would be convenient\n", + "to be able to visualize those tables in some way. We'll write a little helper\n", + "function to accomplish this." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "885c8e65", + "metadata": {}, + "outputs": [], + "source": [ + "def viz_tables(query, *, database=db):\n", + " return pd.read_sql_query(query, database)" + ] + }, + { + "cell_type": "markdown", + "id": "54160eb5", + "metadata": {}, + "source": [ + "Here's how we can use our helper function. It gives a pretty nice visualization\n", + "of our table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5cb6f058", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "viz_tables('''SELECT * FROM candidates''')" + ] + }, + { + "cell_type": "markdown", + "id": "4ee9ebf0", + "metadata": {}, + "source": [ + "<a id='step_1'></a>\n", + "# Step 1\n", + "\n", + "We still need to load the data in the contributors table." + ] + }, + { + "cell_type": "markdown", + "id": "55f108e2", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Load the data in the `contributors` table by loading the data from the\n", + " `contributors.txt` file. **You are not allowed to use a `for`-loop in this\n", + " task (comprehensions are allowed)**.\n", + "* Display the contributors table." + ] + }, + { + "cell_type": "markdown", + "id": "b9abda82", + "metadata": {}, + "source": [ + "<a id='step_2'></a>\n", + "# Step 2: Various Queries\n", + "\n", + "We can query our database for entries with certain characteristics. For\n", + "example, we can query the `candidates` table for entries whose middle name\n", + "fields are not empty." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "973f46a1", + "metadata": {}, + "outputs": [], + "source": [ + "query = '''SELECT * FROM candidates WHERE middle_name <> \"\"'''\n", + "viz_tables(query)" + ] + }, + { + "cell_type": "markdown", + "id": "60807ada", + "metadata": {}, + "source": [ + "We can also see how many entries satisfy the query:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae2e9d22", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"{} candidates have a middle initial.\".format(viz_tables(query).shape[0]))" + ] + }, + { + "cell_type": "markdown", + "id": "9bcb0e32", + "metadata": {}, + "source": [ + "This used the `shape` method on the returned `pandas` table. You'll get to\n", + "practice counting in SQL later." + ] + }, + { + "cell_type": "markdown", + "id": "7397085b", + "metadata": {}, + "source": [ + "### Do the following queries:\n", + "\n", + "* Display the contributors where the state is \"PA\"\n", + "* Display the contributors where the amount contributed is greater than\n", + " $\\$1000.00$.\n", + "* Display the contributors from the state \"UT\" where the amount contributed is\n", + " greater than $\\$1000.00$.\n", + "* Display the contributors who didn't list their state\n", + " - **Hint**: Match `state` to the empty string\n", + "* Display the contributors from \"WA\" or \"PA\"\n", + "* Display the contributors who contributed between $\\$100.00$ and $\\$200.00$.\n", + " - **Hint**: You can use the `BETWEEN 100.00 and 200.00` clause." + ] + }, + { + "cell_type": "markdown", + "id": "79f589bb", + "metadata": {}, + "source": [ + "<a id='step_3'></a>\n", + "# Step 3: Sorting\n", + "\n", + "It could be beneficial to sort by one of the attributes in the database. The\n", + "following cell contains a basic sorting demo. Run it and try to understand what\n", + "happened." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8be6c4fb", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT * FROM candidates ORDER BY id DESC''')" + ] + }, + { + "cell_type": "markdown", + "id": "07fc2df1", + "metadata": {}, + "source": [ + "### Do the following sorts on the specified tables:\n", + "\n", + "* Sort the `candidates` table by `last_name`.\n", + "* Sort the `contributors` table by the `amount` in descending order where `amount`\n", + " is restricted to be between $\\$1000.00$ and $\\$5000.00$.\n", + " - **Hint:** In your SQL command, start with getting the amount between the\n", + " specified range followed by the sort. This will all be done in one line.\n", + "* Sort the contributors who donated between $\\$1000.00$ and $\\$5000.00$ by\n", + " `candidate_id` and then by `amount` in descending order.\n", + " - **Hint**: Multiple orderings can be accomplished by separating requests\n", + " after `ORDER BY` with commas.\n", + " - e.g. `ORDER BY amount ASC, last_name DESC`" + ] + }, + { + "cell_type": "markdown", + "id": "b77f01d8", + "metadata": {}, + "source": [ + "<a id='step_4'></a>\n", + "# Step 4: Selecting Columns\n", + "\n", + "So far, we've been selecting all columns from a table (i.e. `SELECT * FROM`).\n", + "Often, we just want to select specific columns (e.g. `SELECT amount FROM`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99c985ac", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT last_name, party FROM candidates''')" + ] + }, + { + "cell_type": "markdown", + "id": "02f44adb", + "metadata": {}, + "source": [ + "Using the `DISTINCT` clause, you remove duplicate rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8384471d", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT DISTINCT party FROM candidates''')" + ] + }, + { + "cell_type": "markdown", + "id": "9c69c148", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Get the first and last name of contributors. Make sure each row has distinct\n", + " values." + ] + }, + { + "cell_type": "markdown", + "id": "8d7e1ffb", + "metadata": {}, + "source": [ + "<a id='step_5'></a>\n", + "# Step 5: Altering Tables\n", + "\n", + "The `ALTER` clause allows us to modify tables in our database. Here, we add a\n", + "new column to our candidates table called `full_name`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d7eb5c8", + "metadata": {}, + "outputs": [], + "source": [ + "cursor.execute('''ALTER TABLE candidates ADD COLUMN full_name TEXT''')\n", + "viz_tables('''SELECT * FROM candidates''')" + ] + }, + { + "cell_type": "markdown", + "id": "b97cbadb", + "metadata": {}, + "source": [ + "What if we want to rename or delete a column? It can't be done with `SQLite`\n", + "with a single command. We need to follow some roundabout steps (see [`SQLite`\n", + "ALTER TABLE](http://www.sqlitetutorial.net/sqlite-alter-table/)). We won't\n", + "consider this case at the moment." + ] + }, + { + "cell_type": "markdown", + "id": "7d2e3fc3", + "metadata": {}, + "source": [ + "For now, let's put a few commands together to populate the `full_name` column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f2d99fc", + "metadata": {}, + "outputs": [], + "source": [ + "query = '''SELECT id, last_name, first_name FROM candidates''' # Select a few columns\n", + "full_name_and_id = [(attr[1] + \", \" + attr[2], attr[0]) for attr in cursor.execute(query).fetchall()] # List of tuples: (full_name, id)\n", + "\n", + "update = '''UPDATE candidates SET full_name = ? WHERE id = ?''' # Update the table\n", + "for rows in full_name_and_id:\n", + " cursor.execute(update, rows)\n", + "\n", + "query = '''SELECT * FROM candidates'''\n", + "viz_tables(query)" + ] + }, + { + "cell_type": "markdown", + "id": "11680111", + "metadata": {}, + "source": [ + "Here's another update, this time on an existing column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "419f8252", + "metadata": {}, + "outputs": [], + "source": [ + "update = '''UPDATE candidates SET full_name = \"WINNER\" WHERE last_name = \"Obama\"'''\n", + "cursor.execute(update)\n", + "update = '''UPDATE candidates SET full_name = \"RUNNER-UP\" WHERE last_name = \"McCain\"'''\n", + "cursor.execute(update)\n", + "viz_tables(query)" + ] + }, + { + "cell_type": "markdown", + "id": "c83d645a", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Add a new column to the contributors table called `full_name`. The value in\n", + " that column should be in the form `last_name, first_name`.\n", + "* Change the value in the `full_name` column to the string `\"Too Much\"` if\n", + " someone donated more than $\\$1000.00$." + ] + }, + { + "cell_type": "markdown", + "id": "f162d811", + "metadata": {}, + "source": [ + "<a id='step_6'></a>\n", + "# Step 6: Aggregation\n", + "\n", + "You can perform reduction operations on the values in the database. For\n", + "example, you can compute the maximum, minimum, sum or the total number from\n", + "multiple input values. Here's a little example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "becc8cd7", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT *, MAX(amount) AS max_amount FROM contributors''')" + ] + }, + { + "cell_type": "markdown", + "id": "b5332efb", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Modify the demo to only output the max amount.\n", + " * **Hints:**\n", + " - Instead of using `SELECT *, MAX(amount) ...` you can try `SELECT\n", + " MAX(amount) ...`\n", + " - You will want to use `cursor.execute()` here and then `fetchall()`.\n", + " - Do not display your results in a table. It is sufficient to write\n", + " the answer out to the screen as a single number.\n", + "* Count how many donations there were above $\\$1000.00$.\n", + " * **Hint:** There is a `COUNT` function.\n", + "* Calculate the average *positive* donation.\n", + " * **Hint:** There is an `AVG` function.\n", + "* Calculate the average contribution from each state and display in a table.\n", + " Restrict to positive values again.\n", + " - **Hint**: Use code that looks like: `\"SELECT state,SUM(amount) FROM\n", + " contributors GROUP BY state\"`.\n", + "\n", + "<a id='step_7'></a>\n", + "# Step 7: DELETE\n", + "\n", + "We have already noted that `SQLite` can't drop columns in a straightforward\n", + "manner. However, it can delete rows quite simply. Here's the syntax:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e50981bb", + "metadata": {}, + "outputs": [], + "source": [ + "deletion = '''DELETE FROM table_name WHERE condition'''" + ] + }, + { + "cell_type": "markdown", + "id": "17bc148a", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Delete rows in the `contributors` table with last name \"Ahrens\".\n", + "\n", + "<a id='step_8'></a>\n", + "# Step 8: LIMIT\n", + "\n", + "The `LIMIT` clause offers convenient functionality. It allows you to constrain\n", + "the number of rows returned by your query. It shows up in many guises." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4ffeb0f", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT * FROM candidates LIMIT 3''')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7364af8b", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT * FROM candidates LIMIT 4 OFFSET 5''')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "863cabe9", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables('''SELECT * FROM candidates ORDER BY last_name LIMIT 4 OFFSET 5''')" + ] + }, + { + "cell_type": "markdown", + "id": "6dba32f0", + "metadata": {}, + "source": [ + "### Do the following:\n", + "\n", + "* Query and display the ten most generous donors.\n", + "* Query and display the ten least generous donors who donated a positive amount\n", + " of money (since the data we have has some negative numbers in it...)." + ] + }, + { + "cell_type": "markdown", + "id": "caff0ad7", + "metadata": {}, + "source": [ + "# Save\n", + "\n", + "Don't forget to save all of these changes to your database using `db.commit()`.\n", + "Before closing your editor or IDE, be sure to close the database connection with\n", + "`db.close()`." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "md,ipynb" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/lecture23/candidates.txt b/lecture23/candidates.txt @@ -0,0 +1 @@ +../lecture22/candidates.txt +\ No newline at end of file diff --git a/lecture23/contributors.txt b/lecture23/contributors.txt @@ -0,0 +1 @@ +../lecture22/contributors.txt +\ No newline at end of file diff --git a/lecture23/fig/inner_join.png b/lecture23/fig/inner_join.png Binary files differ. diff --git a/lecture23/fig/left_join.png b/lecture23/fig/left_join.png Binary files differ. diff --git a/lecture23/joins.sh b/lecture23/joins.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# File : joins.sh +# Author : Fabian Wermelinger +# Description: Examples for table joins of lecture 23 +# Copyright 2022 Harvard University. All Rights Reserved. + +cat <<EOF | sqlite3 +.header on +.mode column +.nullvalue NULL + +CREATE TABLE A ( +ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, +Name TEXT NOT NULL, +Office TEXT NOT NULL, +Salary REAL +); +INSERT INTO A (Name, Office, Salary) VALUES ('Frank', 'A12', 45000.0); +INSERT INTO A (Name, Office, Salary) VALUES ('Roberta', 'A10', 80000.0); +INSERT INTO A (Name, Office, Salary) VALUES ('Lory', 'B07', 50000.0); + +CREATE TABLE B ( +ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, +Bonus REAL, +EID INTEGER NOT NULL, +FOREIGN KEY (EID) REFERENCES A(ID) +); +INSERT INTO B (Bonus, EID) VALUES (8000.0, 1); +INSERT INTO B (Bonus, EID) VALUES (10000.0, 3); +INSERT INTO B (Bonus, EID) VALUES (1000000.0, 10); + +.print 'Command: SELECT * FROM A INNER JOIN B ON B.EID = A.ID;' +SELECT * FROM A INNER JOIN B ON B.EID = A.ID; +.print '' + +.print 'Command: SELECT * FROM A, B WHERE B.EID = A.ID;' +SELECT * FROM A, B WHERE B.EID = A.ID; +.print '' + +.print 'Command: SELECT * FROM A LEFT OUTER JOIN B ON B.EID = A.ID;' +SELECT * FROM A LEFT OUTER JOIN B ON B.EID = A.ID; +.print '' + +.print 'Command: SELECT * FROM B LEFT OUTER JOIN A ON B.EID = A.ID;' +SELECT * FROM B LEFT OUTER JOIN A ON B.EID = A.ID; +.print '' + +-- How to address columns with identical name? +-- --> alias two columns with identical names to distinct names in joined table +.print 'Command: SELECT A.ID AS AID, B.ID AS BID FROM B LEFT OUTER JOIN A ON B.EID = A.ID;' +SELECT A.ID AS AID, B.ID AS BID FROM B LEFT OUTER JOIN A ON B.EID = A.ID; +.print '' + +-- as opposed to this less useful variant +.print 'Command: SELECT A.ID, B.ID FROM B LEFT OUTER JOIN A ON B.EID = A.ID;' +SELECT A.ID, B.ID FROM B LEFT OUTER JOIN A ON B.EID = A.ID; +EOF diff --git a/lecture23/lecture23.ipynb b/lecture23/lecture23.ipynb @@ -0,0 +1,1174 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8d63b077", + "metadata": {}, + "source": [ + "# Lecture 23\n", + "## Thursday, November 17th 2022\n", + "### Joins with SQLite and Pandas" + ] + }, + { + "cell_type": "markdown", + "id": "bbe7abc2", + "metadata": {}, + "source": [ + "## Starting Up\n", + "\n", + "You can connect to the saved database from Tuesday if you want.\n", + "Alternatively, for extra practice, you can just recreate it from the datasets\n", + "provided in the `.txt` files.\n", + "\n", + "# Exercise Deliverables\n", + "\n", + "> 1. Copy the Jupyter notebook into `lab/pp12` in your private Git repository and\n", + "> commit on your default branch. You should already have the `candidates.txt` and\n", + "> `contributors.txt` data files in this directory from last time.\n", + "> 2. For each exercise in the notebook, there are instructions labeled\n", + "> \"**Do the following:**\". Put all the code\n", + "> for those instructions in _code cell(s) immediately following the\n", + "> instructions_. The code in those cells should be regular Python code.\n", + "> You should place comments where appropriate that describe your intentions.\n", + "> **Note:** To get the\n", + "> Pandas tables to display in a cell, use `display()`.\n", + "> 3. Save and close your database. Be sure to upload your databases in\n", + "> `lab/pp12` as well. Please name your databases **`lecture23.sqlite`**\n", + "> and **`lecture23_pandas.sqlite`**.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee27a7d0", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "import numpy as np\n", + "import pandas as pd\n", + "import time\n", + "\n", + "pd.set_option('display.width', 500)\n", + "pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', 100)\n", + "pd.set_option('display.notebook_repr_html', True)\n", + "\n", + "db = sqlite3.connect('lecture23.sqlite')\n", + "cursor = db.cursor()\n", + "cursor.execute(\"DROP TABLE IF EXISTS candidates\")\n", + "cursor.execute(\"DROP TABLE IF EXISTS contributors\")\n", + "cursor.execute(\"PRAGMA foreign_keys=1\")\n", + "\n", + "cursor.execute(\n", + " '''CREATE TABLE candidates (\n", + " id INTEGER PRIMARY KEY NOT NULL, \n", + " first_name TEXT, \n", + " last_name TEXT, \n", + " middle_name TEXT, \n", + " party TEXT NOT NULL)'''\n", + ")\n", + "\n", + "db.commit() # Commit changes to the database\n", + "\n", + "cursor.execute(\n", + " '''CREATE TABLE contributors (\n", + " id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, \n", + " last_name TEXT, \n", + " first_name TEXT, \n", + " middle_name TEXT, \n", + " street_1 TEXT, \n", + " street_2 TEXT, \n", + " city TEXT, \n", + " state TEXT, \n", + " zip TEXT, \n", + " amount REAL, \n", + " date DATETIME, \n", + " candidate_id INTEGER NOT NULL, \n", + " FOREIGN KEY(candidate_id) REFERENCES candidates(id))'''\n", + ")\n", + "\n", + "db.commit()\n", + "\n", + "with open(\"candidates.txt\") as candidates:\n", + " next(candidates) # jump over the header\n", + " for line in candidates.readlines():\n", + " vals_to_insert = line.strip().split('|')\n", + " cursor.execute(\n", + " '''INSERT INTO candidates \n", + " (id, first_name, last_name, middle_name, party)\n", + " VALUES (?, ?, ?, ?, ?)''', vals_to_insert\n", + " )\n", + "\n", + "with open(\"contributors.txt\") as contributors:\n", + " next(contributors) # jump over the header\n", + " for line in contributors.readlines():\n", + " vals_to_insert = line.strip().split('|')[1:]\n", + " cursor.execute(\n", + " '''INSERT INTO contributors (last_name, first_name, middle_name, \n", + " street_1, street_2, city, state, zip, amount, date, candidate_id) \n", + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',\n", + " vals_to_insert\n", + " )\n", + "\n", + "\n", + "def viz_tables(query, *, database=db):\n", + " return pd.read_sql_query(query, database)\n", + "\n", + "from IPython.display import display" + ] + }, + { + "cell_type": "markdown", + "id": "4ab0105a", + "metadata": {}, + "source": [ + "## Recap\n", + "\n", + "Last time, you played with a number of SQLite commands to query and update the\n", + "tables in the database.\n", + "\n", + "One thing we didn't get to was how to query the contributors table based off\n", + "of a query in the candidates table (parent/child relationships). For example,\n", + "suppose you want to query which contributors donated to Obama. You could use a\n", + "nested `SELECT` statement to accomplish that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2e61c76", + "metadata": {}, + "outputs": [], + "source": [ + "viz_tables(\n", + " '''SELECT * FROM contributors WHERE candidate_id = (SELECT id from candidates WHERE last_name = \"Obama\")'''\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2801fdb8", + "metadata": {}, + "source": [ + "# Joins\n", + "\n", + "The last example involved querying data from multiple tables.\n", + "\n", + "In particular, we combined columns from the two related tables (related through\n", + "the `FOREIGN KEY`).\n", + "\n", + "This leads to the idea of *joining* multiple tables together. SQL has a set\n", + "of commands to handle different types of joins. SQLite does not support the\n", + "full suite of join commands offered by SQL but you should still be able to get\n", + "the main ideas from the limited command set.\n", + "\n", + "We'll begin with the `INNER JOIN`." + ] + }, + { + "cell_type": "markdown", + "id": "c9695740", + "metadata": {}, + "source": [ + "## INNER JOIN\n", + "\n", + "The idea here is that you will combine the tables if the values of certain\n", + "columns are the same between the two tables. In our example, we will join the\n", + "two tables based on the candidate `id`. The result of the `INNER JOIN` will be\n", + "a new table consisting of the columns we requested and containing the common\n", + "data. Since we are joining based off of the candidate `id`, we will not be\n", + "excluding any rows.\n", + "\n", + "### Example\n", + "\n", + "Here are two tables. Table A has the form:\n", + "\n", + "| nA | attr | id |\n", + "|:----:|:-----:|:---:|\n", + "| s1 | 23 | 0 |\n", + "| s2 | 7 | 2 |\n", + "\n", + "and table B has the form:\n", + "\n", + "| nB | attr | id |\n", + "|:----:|:-----:|:---:|\n", + "| t1 | 60 | 0 |\n", + "| t2 | 14 | 7 |\n", + "| t3 | 22 | 2 |\n", + "\n", + "Table A is associated with Table B through a foreign key on the id column.\n", + "\n", + "If we join the two tables by comparing the id columns and selecting the nA, nB,\n", + "and attr columns then we'll get\n", + "\n", + "| nA | A.attr | nB | B.attr |\n", + "|:----:|:-------:|:---:|:------:|\n", + "| s1 | 23 | t1 | 60 |\n", + "| s2 | 7 | t3 | 22 |\n", + "\n", + "\n", + "The SQLite code to do this join would be\n", + "\n", + "```SQL\n", + "SELECT nA, A.attr, nB, B.attr FROM A INNER JOIN B ON B.id = A.id\n", + "```\n", + "\n", + "Notice that the second row in table B is gone because the id values are not the\n", + "same.\n", + "\n", + "### Visualization\n", + "\n", + "What is SQL doing with this operation? It may help to visualize this with a\n", + "Venn diagram. Table A has rows with values corresponding to the `id`\n", + "attribute. Table B has rows with values corresponding to the `id` attribute.\n", + "The `INNER JOIN` will combine the two tables such that rows with common entries\n", + "in the `id` fields are included. We essentially have the following Venn\n", + "diagram.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "1c12c276", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "1. Using an `INNER JOIN`, join the `candidates` and `contributors` tables by\n", + " comparing the `id` column in the `candidates` table with the `candidate_id`\n", + " column in the `contributors` table. Display your joined table with the\n", + " columns `contributors.last_name`, `contributors.first_name`, `amount` and\n", + " `candidates.last_name`.\n", + "2. Do the same inner join as above, but this time append a `WHERE`\n", + " clause to select a specific candidate's last name.\n" + ] + }, + { + "cell_type": "markdown", + "id": "8e462543", + "metadata": {}, + "source": [ + "## `LEFT JOIN` or `LEFT OUTER JOIN`\n", + "\n", + "There are many ways to combine two tables. We just explored one possibility in\n", + "which we combined the tables based upon the intersection of the two tables (the\n", + "`INNER JOIN`).\n", + "\n", + "Now we'll look at the `LEFT JOIN` (or `LEFT OUTER JOIN` in some databases).\n", + "\n", + "In words, the `LEFT JOIN` is combining the tables based upon what is in the\n", + "intersection of the two tables *and* what is in the \"reference\" table (left\n", + "table in the SQL command).\n", + "\n", + "We can consider our toy example in two guises:\n", + "\n", + "#### Example A\n", + "\n", + "Let's do a `LEFT JOIN` of table B from table A. That is, we'd like to make a\n", + "new table by putting table B into table A. In this case, we'll consider table A\n", + "our \"reference\" table. We're comparing by the `id` column again. We know that\n", + "these two tables share ids 0 and 2 and table A doesn't have anything else in it.\n", + "The resulting table is:\n", + "\n", + "| nA | A.attr | nB | B.attr |\n", + "|:----:|:-------:|:---:|:------:|\n", + "| s1 | 23 | t1 | 60 |\n", + "| s2 | 7 | t3 | 22 |\n", + "\n", + "That's not very exciting. It's the same result as from the `INNER JOIN`. We\n", + "can do another example that may be more enlightening.\n", + "\n", + "#### Example B\n", + "\n", + "Let's do a `LEFT JOIN` of table A from table B. That is, we'd like to make a\n", + "new table by putting table A into table B. In this case, we'll consider table B\n", + "our \"reference\" table. Again, we use the `id` column from comparison. We know\n", + "that these two tables share ids 0 and 2. This time, table B also contains the\n", + "id 7, which is not shared by table A. The resulting table is:\n", + "\n", + "| nA | A.attr | nB | B.attr |\n", + "|:----:|:-------:|:---:|:------:|\n", + "| s1 | 23 | t1 | 60 |\n", + "| NULL | NULL | t2 | 14 |\n", + "| s2 | 7 | t3 | 22 |\n", + "\n", + "Notice that SQLite filled in the missing entries for us. This is necessary\n", + "for completion of the requested join.\n", + "\n", + "The SQLite commands to accomplish all of this are:\n", + "\n", + "```SQL\n", + "SELECT nA, A.attr, nB, B.attr FROM A LEFT JOIN B ON B.id = A.id\n", + "```\n", + "\n", + "and\n", + "\n", + "```SQL\n", + "SELECT nA, A.attr, nB, B.attr FROM B LEFT JOIN A ON A.id = B.id\n", + "```\n", + "\n", + "Here is a visualization using Venn diagrams of the `LEFT JOIN`.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "e2146d49", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "Use the following two tables to do the first two exercises in this section.\n", + "Table A has the form:\n", + "\n", + "| nA | attr | id |\n", + "|:----:|:-----:|:---:|\n", + "| s1 | 23 | 0 |\n", + "| s2 | 7 | 2 |\n", + "| s3 | 15 | 3 |\n", + "| s4 | 31 | 7 |\n", + "\n", + "and table B has the form:\n", + "\n", + "| nB | attr | id |\n", + "|:----:|:-----:|:---:|\n", + "| t1 | 60 | 0 |\n", + "| t2 | 14 | 7 |\n", + "| t3 | 22 | 2 |\n", + "\n", + "1. Write the markdown table that would result from a `LEFT JOIN` using table A\n", + " as the reference and the `id` columns for comparison. Example with arbitrary\n", + " column names:\n", + " ```md\n", + " | Col1 | Col2 | Col3 | Col4 |\n", + " |:----:|:----:|:----:|:----:|\n", + " | val1 | val2 | val3 | val4 |\n", + " | val5 | val6 | val7 | val8 |\n", + " ```\n", + "2. Write the markdown table that would result from a `LEFT JOIN` using table B\n", + " as the reference and the `id` columns for comparison.\n", + "3. Now back to the candidates and their contributors. Create a new table with\n", + " the following form:\n", + "\n", + " | average contribution | candidate last name |\n", + " |:--------------------:|:-------------------:|\n", + " | ... | ... |\n", + "\n", + " The table should be created using the `LEFT JOIN` clause on the\n", + " `contributors` table by joining the `candidates` table using the `id`\n", + " column. The `average contribution` column should be obtained using the\n", + " `AVG()` SQL function. Use the `GROUP BY` clause on the `candidates` last\n", + " name.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "7a608a69", + "metadata": {}, + "source": [ + "# Pandas" + ] + }, + { + "cell_type": "markdown", + "id": "e9962018", + "metadata": {}, + "source": [ + "We've been working with databases for the last few lectures and learning\n", + "SQLite commands to work with and manipulate the SQL databases. Pandas is\n", + "a powerful Python package that provides broad support for data structures. It\n", + "can be used to interact with relational databases through its own methods and\n", + "even through SQL commands.\n", + "\n", + "> In the last part of this lecture, you will get to redo a number of the\n", + "> previous database exercises using Pandas.\n", + "\n", + "We won't be able to cover Pandas from the ground up, but it's a\n", + "well-documented library and is fairly easy to get up and running. The website\n", + "can be found at the following link: [Pandas](http://pandas.pydata.org/). A\n", + "very good reference for Pandas is the book [\"Python for data\n", + "analysis\"](https://www.amazon.com/gp/product/1491957662/ref=as_li_tl?ie=UTF8&tag=quantpytho-20&camp=1789&creative=9325&linkCode=as2&creativeASIN=1491957662&linkId=8c3bf87b221dbcd8f541f0db20d4da83)\n", + "by Wes McKinney, the creator of Pandas himself." + ] + }, + { + "cell_type": "markdown", + "id": "c3b3f76d", + "metadata": {}, + "source": [ + "## Reading a datafile into Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "daf5676d", + "metadata": {}, + "outputs": [], + "source": [ + "# Using Pandas naming convention\n", + "dfcand = pd.read_csv(\"candidates.txt\", sep=\"|\")\n", + "dfcand" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38244660", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr = pd.read_csv(\"contributors.txt\", sep=\"|\")\n", + "dfcontr" + ] + }, + { + "cell_type": "markdown", + "id": "1f989838", + "metadata": {}, + "source": [ + "Reading things in is quite easy with Pandas. Notice that Pandas populates\n", + "empty fields with `NaN` values. The `id` column in the `contributors` dataset is\n", + "superfluous. Let's delete it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c69cbd93", + "metadata": {}, + "outputs": [], + "source": [ + "del dfcontr['id']\n", + "dfcontr.head()" + ] + }, + { + "cell_type": "markdown", + "id": "1177054b", + "metadata": {}, + "source": [ + "Very nice! And we used the `head` method to print out the first five rows." + ] + }, + { + "cell_type": "markdown", + "id": "372c68d9", + "metadata": {}, + "source": [ + "## Creating a Table with Pandas\n", + "\n", + "We can use Pandas to create tables in a database. First, let's create a new\n", + "(empty) SQLite database:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4830fb87", + "metadata": {}, + "outputs": [], + "source": [ + "# commit the previous work\n", + "db.commit()\n", + "\n", + "# new database for Pandas operations\n", + "db = sqlite3.connect('lecture23_pandas.sqlite')\n", + "cursor = db.cursor()\n", + "cursor.execute(\"DROP TABLE IF EXISTS candidates\")\n", + "cursor.execute(\"DROP TABLE IF EXISTS contributors\")\n", + "cursor.execute(\"PRAGMA foreign_keys=1\")\n", + "\n", + "cursor.execute(\n", + " '''CREATE TABLE candidates (\n", + " id INTEGER PRIMARY KEY NOT NULL, \n", + " first_name TEXT, \n", + " last_name TEXT, \n", + " middle_name TEXT, \n", + " party TEXT NOT NULL)'''\n", + ")\n", + "\n", + "db.commit() # Commit changes to the database\n", + "\n", + "cursor.execute(\n", + " '''CREATE TABLE contributors (\n", + " id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, \n", + " last_name TEXT, \n", + " first_name TEXT, \n", + " middle_name TEXT, \n", + " street_1 TEXT, \n", + " street_2 TEXT, \n", + " city TEXT, \n", + " state TEXT, \n", + " zip TEXT, \n", + " amount REAL, \n", + " date DATETIME, \n", + " candidate_id INTEGER NOT NULL, \n", + " FOREIGN KEY(candidate_id) REFERENCES candidates(id))'''\n", + ")\n", + "\n", + "db.commit()" + ] + }, + { + "cell_type": "markdown", + "id": "3d335768", + "metadata": {}, + "source": [ + "Last time, we opened the data files with Python and then manually used\n", + "SQLite commands to populate the individual tables. We can use Pandas\n", + "instead like so:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ef39d41", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.to_sql(\"candidates\", db, if_exists=\"append\", index=False)" + ] + }, + { + "cell_type": "markdown", + "id": "74e41652", + "metadata": {}, + "source": [ + "What is the size of our table?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dda8ab8a", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.shape" + ] + }, + { + "cell_type": "markdown", + "id": "bccda7d1", + "metadata": {}, + "source": [ + "We can visualize the data in our Pandas-populated SQL table. No surprises\n", + "here except that Pandas did everything for us (contrast this to our manual\n", + "file read last time using `for`-loops and list comprehensions)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7205d082", + "metadata": {}, + "outputs": [], + "source": [ + "display(viz_tables('''SELECT * FROM candidates'''))" + ] + }, + { + "cell_type": "markdown", + "id": "6463c34e", + "metadata": {}, + "source": [ + "## Querying a table with Pandas" + ] + }, + { + "cell_type": "markdown", + "id": "09479688", + "metadata": {}, + "source": [ + "### One Way\n", + "\n", + "Using the `query` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4dfdae96", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.query(\"first_name=='Mike' & party=='D'\")" + ] + }, + { + "cell_type": "markdown", + "id": "8d5ac2e6", + "metadata": {}, + "source": [ + "### Another Way\n", + "\n", + "Using the `__getitem__` special method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0ff20ba", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[(dfcand.first_name==\"Mike\") & (dfcand.party==\"D\")]" + ] + }, + { + "cell_type": "markdown", + "id": "d0443eef", + "metadata": {}, + "source": [ + "### More Queries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e9d78ac", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[dfcand.middle_name.notnull()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd900a67", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[dfcand.first_name.isin(['Mike', 'Hillary'])]" + ] + }, + { + "cell_type": "markdown", + "id": "55a51695", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "1. Use Pandas to populate the contributors table in the SQLite database and\n", + " display the SQL table with the `viz_tables` helper.\n", + "2. Query the `contributors` Pandas `DataFrame` with the following constraints:\n", + " 1. List entries where the state is \"VA\" and the amount is less than\n", + " $\\$400.00$.\n", + " 2. List entries where the state is \"NULL\".\n", + " 3. List entries for the states of Texas and Pennsylvania.\n", + " 4. List entries where the amount contributed is between $\\$10.00$ and\n", + " $\\$50.00$." + ] + }, + { + "cell_type": "markdown", + "id": "0b46cc6e", + "metadata": {}, + "source": [ + "## Sorting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c5a40bf", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.sort_values(by='party')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "499d7aa0", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.sort_values(by='party', ascending=False)" + ] + }, + { + "cell_type": "markdown", + "id": "4364b1bc", + "metadata": {}, + "source": [ + "## Selecting Columns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "baf9374f", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[['last_name', 'party']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1232473", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[['last_name', 'party']].count()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfd06fa6", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[['first_name']].drop_duplicates()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "830863f8", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[['first_name']].drop_duplicates().count()" + ] + }, + { + "cell_type": "markdown", + "id": "1a5e16fa", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "Use the `contributors` Pandas `DataFrame` and the `display` helper to\n", + "display the results.\n", + "\n", + "1. Sort the `DataFrame` by `amount` and order in *descending* order.\n", + "2. Select the `first_name` and `amount` columns.\n", + "3. Select the `last_name` and `first_name` columns and drop duplicates.\n", + "4. Count how many there are after the duplicates have been dropped." + ] + }, + { + "cell_type": "markdown", + "id": "89e753b0", + "metadata": {}, + "source": [ + "## Altering Tables" + ] + }, + { + "cell_type": "markdown", + "id": "dc563f80", + "metadata": {}, + "source": [ + "Creating a new column is quite easy with Pandas." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c565bc3", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand['name'] = dfcand['last_name'] + \", \" + dfcand['first_name']\n", + "dfcand" + ] + }, + { + "cell_type": "markdown", + "id": "6933f2cd", + "metadata": {}, + "source": [ + "We can change an existing field as well (see the [`loc`\n", + "method](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.loc.html)):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "beed2150", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.loc[dfcand.first_name == \"Mike\", \"name\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07415edc", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.loc[dfcand.first_name == \"Mike\", \"name\"] = \"Mikey\"\n", + "dfcand" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8f5ce9c", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.query(\"first_name == 'Mike'\")\n", + "dfcand" + ] + }, + { + "cell_type": "markdown", + "id": "0c1ef2e3", + "metadata": {}, + "source": [ + "You may recall that SQLite doesn't have the functionality to drop a column.\n", + "It can be done in one line using Pandas:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5055a447", + "metadata": {}, + "outputs": [], + "source": [ + "del dfcand['name']\n", + "dfcand" + ] + }, + { + "cell_type": "markdown", + "id": "d8843e4a", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "Use the `contributors` Pandas `DataFrame` and the `display` helper to\n", + "display the results.\n", + "\n", + "1. Create a `name` column for the `contributors` table with field entries of the\n", + " form \"last name, first name\"\n", + "2. For contributors from the state of \"PA\", change `name` to \"X\".\n", + "3. Delete the newly created name column." + ] + }, + { + "cell_type": "markdown", + "id": "b0080e20", + "metadata": {}, + "source": [ + "## Aggregation\n", + "\n", + "We'd like to get information about the tables such as the maximum amount\n", + "contributed to the candidates. Basic statistics on a Pandas frame can be\n", + "obtained using the `describe()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66937b4b", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand.describe()" + ] + }, + { + "cell_type": "markdown", + "id": "4bfe4c7f", + "metadata": {}, + "source": [ + "It's not very interesting with the candidates table because the candidates table\n", + "only has one numeric column. Here are a few more data queries using the\n", + "contributors table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfecf3cf", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr.amount.max()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b148b1d", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr[dfcontr.amount==dfcontr.amount.max()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6515bac7", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr.groupby(\"state\").sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3759b090", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr.groupby(\"state\")[\"amount\"].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6a0175d", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr.state.unique()" + ] + }, + { + "cell_type": "markdown", + "id": "c727e2fd", + "metadata": {}, + "source": [ + "There is also a version of the `LIMIT` clause in SQL. It's very intuitive\n", + "using Pandas in Python:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c2b94df", + "metadata": {}, + "outputs": [], + "source": [ + "dfcand[0:3]" + ] + }, + { + "cell_type": "markdown", + "id": "0b61964d", + "metadata": {}, + "source": [ + "The usual Python slicing works just fine!\n" + ] + }, + { + "cell_type": "markdown", + "id": "9b1c9e3d", + "metadata": {}, + "source": [ + "## Do the following:\n", + "\n", + "Use the `describe()` method on the `contributors` table." + ] + }, + { + "cell_type": "markdown", + "id": "cc537cb1", + "metadata": {}, + "source": [ + "## Joins with Pandas" + ] + }, + { + "cell_type": "markdown", + "id": "5d6a729a", + "metadata": {}, + "source": [ + "Pandas has some documentation on `joins`: [Merge, join, and\n", + "concatenate](http://pandas.pydata.org/pandas-docs/stable/merging.html). If you\n", + "want some more reinforcement on the concepts from earlier regarding `JOIN`, then\n", + "the Pandas documentation may be a good place to get it.\n", + "\n", + "You may also be interested in [a comparison with\n", + "SQL](http://pandas.pydata.org/pandas-docs/stable/comparison_with_sql.html#compare-with-sql-join).\n", + "\n", + "> To do joins with Pandas, we use the `merge` method.\n", + "\n", + "Here's an example of an explicit inner join:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f956838", + "metadata": {}, + "outputs": [], + "source": [ + "cols_wanted = ['last_name_x', 'first_name_x', 'candidate_id', 'id', 'last_name_y', 'amount']\n", + "dfcontr.merge(dfcand, left_on=\"candidate_id\", right_on=\"id\")[cols_wanted]" + ] + }, + { + "cell_type": "markdown", + "id": "842f3710", + "metadata": {}, + "source": [ + "Somewhat more organized with additional grouping and description of resulting\n", + "data frame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7322f44e", + "metadata": {}, + "outputs": [], + "source": [ + "dfcontr.merge(dfcand, left_on=\"candidate_id\", right_on=\"id\")[cols_wanted].groupby('last_name_y').describe()" + ] + }, + { + "cell_type": "markdown", + "id": "79bcc62c", + "metadata": {}, + "source": [ + "### Other Joins with Pandas\n", + "\n", + "We didn't cover all possible joins because SQLite can only handle the few that\n", + "we did discuss. As mentioned, there are workarounds for some things in\n", + "SQLite, but not everything. Fortunately, Pandas can handle pretty much\n", + "everything. Here are a few joins that Pandas can handle:\n", + "\n", + "* `LEFT OUTER`: discussed above\n", + "* `RIGHT OUTER`: think of the \"opposite\" of a `LEFT OUTER` join (shade the\n", + " intersection and *right* set in the Venn diagram).\n", + "* `FULL OUTER`: combine everything from both tables (shade the entire Venn\n", + " diagram)\n", + "\n", + "Lets build the tables from the join exercise above in Pandas:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7eea35c", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "df_A = pd.DataFrame(\n", + " {\n", + " 'nA': ['s1', 's2', 's3', 's4'],\n", + " 'attr': [23, 7, 15, 31],\n", + " 'id': [0, 2, 3, 7]\n", + " }\n", + ")\n", + "\n", + "df_B = pd.DataFrame(\n", + " {\n", + " 'nB': ['t1', 't2', 't3'],\n", + " 'attr': [60, 14, 22],\n", + " 'id': [0, 7, 2]\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "98fcc8c4", + "metadata": {}, + "source": [ + "#### Left Outer Join with Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0dd2148", + "metadata": {}, + "outputs": [], + "source": [ + "df_A.merge(df_B, left_on='id', right_on='id', how='left')" + ] + }, + { + "cell_type": "markdown", + "id": "862e7b92", + "metadata": {}, + "source": [ + "#### Right Outer Join with Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8dd4523", + "metadata": {}, + "outputs": [], + "source": [ + "df_A.merge(df_B, left_on='id', right_on='id', how='right')" + ] + }, + { + "cell_type": "markdown", + "id": "53eceb9e", + "metadata": {}, + "source": [ + "#### Full Outer Join with Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "531b99f2", + "metadata": {}, + "outputs": [], + "source": [ + "df_A.merge(df_B, left_on='id', right_on='id', how='outer')" + ] + }, + { + "cell_type": "markdown", + "id": "2c8822c6", + "metadata": {}, + "source": [ + "# Save our databases\n", + "\n", + "Commit the changes to the open SQL databases and close them. Well done!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d994c14", + "metadata": {}, + "outputs": [], + "source": [ + "db.commit()\n", + "db.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/lecture24/cProfile/.gitignore b/lecture24/cProfile/.gitignore @@ -0,0 +1 @@ +profile diff --git a/lecture24/cProfile/my_module.py b/lecture24/cProfile/my_module.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import time + +def fast(): + time.sleep(0.5) + +def slow(): + time.sleep(1) + +def main(): + for i in range(5): + fast() + for i in range(3): + slow() + +if __name__ == "__main__": + main() diff --git a/lecture24/cProfile/run_profile.sh b/lecture24/cProfile/run_profile.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +python -m cProfile -o profile \ + my_module.py diff --git a/lecture24/cProfile/run_pstats.sh b/lecture24/cProfile/run_pstats.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +python -m pstats profile diff --git a/lecture24/cProfile_newton/.gitignore b/lecture24/cProfile_newton/.gitignore @@ -0,0 +1,2 @@ +exact +approx_* diff --git a/lecture24/cProfile_newton/newton.py b/lecture24/cProfile_newton/newton.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# File : newton.py +# Description: Newton's method with exact and FD approximated Jacobian +# Copyright 2022 Harvard University. All Rights Reserved. +import cProfile +import numpy as np + +f = lambda x: x - np.exp(-2.0 * np.sin(4.0 * x) * np.sin(4.0 * x)) +J = lambda x: 1.0 + 16.0 * np.exp(-2.0 * np.sin(4.0 * x)**2) * np.sin(4.0 * x) * np.cos(4.0 * x) + +def J_fd(eps): + """Returns finite-difference function object for given discretization `eps`.""" + return lambda x: (f(x + eps) - f(x)) / eps + + +def newton(f, J, x_k, tol=1.0e-8, max_it=100): + """Newton-Raphson method.""" + root = None + for k in range(max_it): + dx_k = -f(x_k) / J(x_k) + if abs(dx_k) < tol: + root = x_k + dx_k + print(f"Found root {root:e} at iteration {k+1}") + break + print(f"Iteration {k+1}: Delta x = {dx_k:e}") + x_k += dx_k + return root + + +def main(J): + """ + Main function with specific Jacobian `J`. + + Parameters + ---------- + J : function + Jacobian function object to be used in the Newton-Raphson method. + + """ + newton(f, J, 0.1, 1.0e-8, 100) + + +if __name__ == "__main__": + # TODO: implement the three cProfile calls here: + # 1. Profile the exact Jacobian and save it in file `exact` + # 2. Profile the FD Jacobian with eps=1.0e-1 and save it in file `approx_coarse` + # 3. Profile the FD Jacobian with eps=1.0e-8 and save it in file `approx_fine` + pass diff --git a/lecture24/cProfile_newton/post.py b/lecture24/cProfile_newton/post.py @@ -0,0 +1,7 @@ +import pstats + +if __name__ == "__main__": + files = ['exact', 'approx_coarse', 'approx_fine'] + for f in files: + profile = pstats.Stats(f) + profile.sort_stats('tottime').print_stats(3) diff --git a/lecture24/gdb_factorial/.gitignore b/lecture24/gdb_factorial/.gitignore @@ -0,0 +1,2 @@ +main +*.o diff --git a/lecture24/gdb_factorial/Makefile b/lecture24/gdb_factorial/Makefile @@ -0,0 +1,11 @@ +CXX ?= g++ +.PHONY: clean + +main: main.cpp factorial.o + $(CXX) -o $@ -g -O0 $^ + +%.o: %.cpp + $(CXX) -o $@ -g -O0 -c $< + +clean: + rm -f main *.o diff --git a/lecture24/gdb_factorial/factorial.cpp b/lecture24/gdb_factorial/factorial.cpp @@ -0,0 +1,13 @@ +// File : factorial.cpp +// Created : Fri Nov 19 2021 03:33:24 PM (-0500) +// Author : Fabian Wermelinger +// Description: Simple recursive function calls (factorial) +// Copyright 2021 Harvard University. All Rights Reserved. + +int factorial(int x) // factorial +{ + if (x > 1) { + return x * factorial(x - 1); // recursive call + } + return 1; +} diff --git a/lecture24/gdb_factorial/main.cpp b/lecture24/gdb_factorial/main.cpp @@ -0,0 +1,8 @@ +// File : main.cpp +// Created : Fri Nov 19 2021 03:33:24 PM (-0500) +// Author : Fabian Wermelinger +// Description: Simple recursive function main function +// Copyright 2021 Harvard University. All Rights Reserved. + +int factorial(int x); +int main(int argc, char *argv[]) { return factorial(5); } diff --git a/lecture24/pdb_factorial/factorial.py b/lecture24/pdb_factorial/factorial.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + + +def factorial(x): + if x > 1: + return x * factorial(x - 1) # recursive call + return 1 + + +def main(): + return factorial(5) + + +if __name__ == "__main__": + main() diff --git a/lecture24/performance/code_object/code_object.py b/lecture24/performance/code_object/code_object.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import dis + +a = 1 +b = 2 + +# source to be compiled into a code object +source = 'a + 1' +# source = 'a + b' + +# compile code object +co = compile(source, '<string>', mode='eval') + +# disassembly columns are: +# line_number byte_offset instruction_name argument_value +dis.dis(co) + +# names and constants +print(f'names: {co.co_names}') +print(f'constants: {co.co_consts}') + +# opcodes and opargs in binary +print(f'raw binary bytecode: {co.co_code}') +h = co.co_code.hex() +inst_code = " ".join([h[i:i + 4] for i in range(0, len(h), 4)]).split() +print(f'hex representation: {" ".join(inst_code)}') + +for code in inst_code: + opcode = f'0x{code[0:2]}' + oparg = f'0x{code[2:4]}' + print(f'opname lookup: {opcode} -> {dis.opname[int(opcode, 16)]}') diff --git a/lecture24/performance/opcode_oparg/opcode_oparg.py b/lecture24/performance/opcode_oparg/opcode_oparg.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 + +def f(): + x0 = 0.0 + x1 = 1.1 + x2 = 2.2 + x3 = 3.3 + x4 = 4.4 + x5 = 5.5 + x6 = 6.6 + x7 = 7.7 + x8 = 8.8 + x9 = 9.9 + x10 = 10.10 + x11 = 11.11 + x12 = 12.12 + x13 = 13.13 + x14 = 14.14 + x15 = 15.15 + x16 = 16.16 + x17 = 17.17 + x18 = 18.18 + x19 = 19.19 + x20 = 20.20 + x21 = 21.21 + x22 = 22.22 + x23 = 23.23 + x24 = 24.24 + x25 = 25.25 + x26 = 26.26 + x27 = 27.27 + x28 = 28.28 + x29 = 29.29 + x30 = 30.30 + x31 = 31.31 + x32 = 32.32 + x33 = 33.33 + x34 = 34.34 + x35 = 35.35 + x36 = 36.36 + x37 = 37.37 + x38 = 38.38 + x39 = 39.39 + x40 = 40.40 + x41 = 41.41 + x42 = 42.42 + x43 = 43.43 + x44 = 44.44 + x45 = 45.45 + x46 = 46.46 + x47 = 47.47 + x48 = 48.48 + x49 = 49.49 + x50 = 50.50 + x51 = 51.51 + x52 = 52.52 + x53 = 53.53 + x54 = 54.54 + x55 = 55.55 + x56 = 56.56 + x57 = 57.57 + x58 = 58.58 + x59 = 59.59 + x60 = 60.60 + x61 = 61.61 + x62 = 62.62 + x63 = 63.63 + x64 = 64.64 + x65 = 65.65 + x66 = 66.66 + x67 = 67.67 + x68 = 68.68 + x69 = 69.69 + x70 = 70.70 + x71 = 71.71 + x72 = 72.72 + x73 = 73.73 + x74 = 74.74 + x75 = 75.75 + x76 = 76.76 + x77 = 77.77 + x78 = 78.78 + x79 = 79.79 + x80 = 80.80 + x81 = 81.81 + x82 = 82.82 + x83 = 83.83 + x84 = 84.84 + x85 = 85.85 + x86 = 86.86 + x87 = 87.87 + x88 = 88.88 + x89 = 89.89 + x90 = 90.90 + x91 = 91.91 + x92 = 92.92 + x93 = 93.93 + x94 = 94.94 + x95 = 95.95 + x96 = 96.96 + x97 = 97.97 + x98 = 98.98 + x99 = 99.99 + x100 = 100.100 + x101 = 101.101 + x102 = 102.102 + x103 = 103.103 + x104 = 104.104 + x105 = 105.105 + x106 = 106.106 + x107 = 107.107 + x108 = 108.108 + x109 = 109.109 + x110 = 110.110 + x111 = 111.111 + x112 = 112.112 + x113 = 113.113 + x114 = 114.114 + x115 = 115.115 + x116 = 116.116 + x117 = 117.117 + x118 = 118.118 + x119 = 119.119 + x120 = 120.120 + x121 = 121.121 + x122 = 122.122 + x123 = 123.123 + x124 = 124.124 + x125 = 125.125 + x126 = 126.126 + x127 = 127.127 + x128 = 128.128 + x129 = 129.129 + x130 = 130.130 + x131 = 131.131 + x132 = 132.132 + x133 = 133.133 + x134 = 134.134 + x135 = 135.135 + x136 = 136.136 + x137 = 137.137 + x138 = 138.138 + x139 = 139.139 + x140 = 140.140 + x141 = 141.141 + x142 = 142.142 + x143 = 143.143 + x144 = 144.144 + x145 = 145.145 + x146 = 146.146 + x147 = 147.147 + x148 = 148.148 + x149 = 149.149 + x150 = 150.150 + x151 = 151.151 + x152 = 152.152 + x153 = 153.153 + x154 = 154.154 + x155 = 155.155 + x156 = 156.156 + x157 = 157.157 + x158 = 158.158 + x159 = 159.159 + x160 = 160.160 + x161 = 161.161 + x162 = 162.162 + x163 = 163.163 + x164 = 164.164 + x165 = 165.165 + x166 = 166.166 + x167 = 167.167 + x168 = 168.168 + x169 = 169.169 + x170 = 170.170 + x171 = 171.171 + x172 = 172.172 + x173 = 173.173 + x174 = 174.174 + x175 = 175.175 + x176 = 176.176 + x177 = 177.177 + x178 = 178.178 + x179 = 179.179 + x180 = 180.180 + x181 = 181.181 + x182 = 182.182 + x183 = 183.183 + x184 = 184.184 + x185 = 185.185 + x186 = 186.186 + x187 = 187.187 + x188 = 188.188 + x189 = 189.189 + x190 = 190.190 + x191 = 191.191 + x192 = 192.192 + x193 = 193.193 + x194 = 194.194 + x195 = 195.195 + x196 = 196.196 + x197 = 197.197 + x198 = 198.198 + x199 = 199.199 + x200 = 200.200 + x201 = 201.201 + x202 = 202.202 + x203 = 203.203 + x204 = 204.204 + x205 = 205.205 + x206 = 206.206 + x207 = 207.207 + x208 = 208.208 + x209 = 209.209 + x210 = 210.210 + x211 = 211.211 + x212 = 212.212 + x213 = 213.213 + x214 = 214.214 + x215 = 215.215 + x216 = 216.216 + x217 = 217.217 + x218 = 218.218 + x219 = 219.219 + x220 = 220.220 + x221 = 221.221 + x222 = 222.222 + x223 = 223.223 + x224 = 224.224 + x225 = 225.225 + x226 = 226.226 + x227 = 227.227 + x228 = 228.228 + x229 = 229.229 + x230 = 230.230 + x231 = 231.231 + x232 = 232.232 + x233 = 233.233 + x234 = 234.234 + x235 = 235.235 + x236 = 236.236 + x237 = 237.237 + x238 = 238.238 + x239 = 239.239 + x240 = 240.240 + x241 = 241.241 + x242 = 242.242 + x243 = 243.243 + x244 = 244.244 + x245 = 245.245 + x246 = 246.246 + x247 = 247.247 + x248 = 248.248 + x249 = 249.249 + x250 = 250.250 + x251 = 251.251 + x252 = 252.252 + x253 = 253.253 + x254 = 254.254 + x255 = 255.255 + x256 = 256.256 + x257 = 257.257 + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20 + x21 + x22 + x23 + x24 + x25 + x26 + x27 + x28 + x29 + x30 + x31 + x32 + x33 + x34 + x35 + x36 + x37 + x38 + x39 + x40 + x41 + x42 + x43 + x44 + x45 + x46 + x47 + x48 + x49 + x50 + x51 + x52 + x53 + x54 + x55 + x56 + x57 + x58 + x59 + x60 + x61 + x62 + x63 + x64 + x65 + x66 + x67 + x68 + x69 + x70 + x71 + x72 + x73 + x74 + x75 + x76 + x77 + x78 + x79 + x80 + x81 + x82 + x83 + x84 + x85 + x86 + x87 + x88 + x89 + x90 + x91 + x92 + x93 + x94 + x95 + x96 + x97 + x98 + x99 + x100 + x101 + x102 + x103 + x104 + x105 + x106 + x107 + x108 + x109 + x110 + x111 + x112 + x113 + x114 + x115 + x116 + x117 + x118 + x119 + x120 + x121 + x122 + x123 + x124 + x125 + x126 + x127 + x128 + x129 + x130 + x131 + x132 + x133 + x134 + x135 + x136 + x137 + x138 + x139 + x140 + x141 + x142 + x143 + x144 + x145 + x146 + x147 + x148 + x149 + x150 + x151 + x152 + x153 + x154 + x155 + x156 + x157 + x158 + x159 + x160 + x161 + x162 + x163 + x164 + x165 + x166 + x167 + x168 + x169 + x170 + x171 + x172 + x173 + x174 + x175 + x176 + x177 + x178 + x179 + x180 + x181 + x182 + x183 + x184 + x185 + x186 + x187 + x188 + x189 + x190 + x191 + x192 + x193 + x194 + x195 + x196 + x197 + x198 + x199 + x200 + x201 + x202 + x203 + x204 + x205 + x206 + x207 + x208 + x209 + x210 + x211 + x212 + x213 + x214 + x215 + x216 + x217 + x218 + x219 + x220 + x221 + x222 + x223 + x224 + x225 + x226 + x227 + x228 + x229 + x230 + x231 + x232 + x233 + x234 + x235 + x236 + x237 + x238 + x239 + x240 + x241 + x242 + x243 + x244 + x245 + x246 + x247 + x248 + x249 + x250 + x251 + x252 + x253 + x254 + x255 + x256 + x257 + + +import dis +import binascii + +dis.dis(f) +h = f.__code__.co_code.hex() +inst_code = " ".join([h[i:i + 4] for i in range(0, len(h), 4)]).split() +print('') +print( + 'Bytes in bytecode:', len(f.__code__.co_code), + f'(number of instructions: {len(inst_code)})' +) +print('Number of elements in `co_const` tuple:', len(f.__code__.co_consts)) +print( + 'Instructions at byte offset 1016 to 1034:\n\n\t', + " ".join(inst_code[1016 // 2:1032 // 2 + 1]) +) +print(f'\n({dis.opname[0x64]:<12s}) 0x64 0xff') +print(f'({dis.opname[0x7d]:<12s}) 0x7d 0xfe') +print(f'({dis.opname[0x90]:<12s}) 0x90 0x01') +print(f'({dis.opname[0x64]:<12s}) 0x64 0x00') +print(f'({dis.opname[0x7d]:<12s}) 0x7d 0xff') +print(f'({dis.opname[0x90]:<12s}) 0x90 0x01') +print(f'({dis.opname[0x64]:<12s}) 0x64 0x01') +print(f'({dis.opname[0x90]:<12s}) 0x90 0x01') +print(f'({dis.opname[0x7d]:<12s}) 0x7d 0x00')
