diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 199547432be29..9e9ce2108ed81 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,7 +1,7 @@ variables: coverage: false -trigger: ['main', '3.10', '3.9', '3.8', '3.7'] +trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index b96a192005a42..c3ecc67057280 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,7 +1,7 @@ variables: coverage: false -pr: ['main', '3.10', '3.9', '3.8', '3.7'] +pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 62ee6f89cda67..0aea3983fa600 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -98,7 +98,7 @@ Lib/ast.py @isidentical /Lib/unittest/test/testmock/* @cjw296 # SQLite 3 -**/*sqlite* @berkerpeksag +**/*sqlite* @berkerpeksag @erlend-aasland # subprocess /Lib/subprocess.py @gpshead diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b489335903772..8de17345df6fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: push: branches: - 'main' + - '3.11' - '3.10' - '3.9' - '3.8' @@ -15,6 +16,7 @@ on: pull_request: branches: - 'main' + - '3.11' - '3.10' - '3.9' - '3.8' @@ -54,6 +56,28 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true fi + check_abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + run: make check-abidump + check_generated_files: name: 'Check if generated files are up to date' runs-on: ubuntu-latest diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index cba1e51ef27d3..ec18735e9b9fa 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -5,6 +5,7 @@ on: push: branches: - 'main' + - '3.11' - '3.10' - '3.9' - '3.8' @@ -14,6 +15,7 @@ on: pull_request: branches: - 'main' + - '3.11' - '3.10' - '3.9' - '3.8' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 3ed66e74b8954..8c4a034896126 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -5,6 +5,7 @@ on: #push: # branches: # - 'main' + # - '3.11' # - '3.10' # - '3.9' # - '3.8' @@ -14,6 +15,7 @@ on: pull_request: branches: - 'main' + - '3.11' - '3.10' - '3.9' - '3.8' @@ -41,12 +43,12 @@ jobs: # Run "check doctest html" as 3 steps to get a more readable output # in the web UI - name: 'Check documentation' - run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" check + run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going" check # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going" doctest - name: 'Build HTML documentation' - run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" html + run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going" html - name: 'Upload' uses: actions/upload-artifact@v3 with: diff --git a/Doc/Makefile b/Doc/Makefile index 3a3417bf99af3..5b6a95813abee 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -18,7 +18,7 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j auto \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) .PHONY: help build html htmlhelp latex text texinfo changes linkcheck \ @@ -213,8 +213,10 @@ dist: rm dist/python-$(DISTVERSION)-docs-texinfo.tar check: - $(SPHINXLINT) -i tools -i $(VENVDIR) -i README.rst - $(SPHINXLINT) ../Misc/NEWS.d/next/ + # Check the docs and NEWS files with sphinx-lint. + # Ignore the tools and venv dirs and check that the default role is not used. + $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role + $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/ serve: @echo "The serve target was removed, use htmlview instead (see bpo-36329)" diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 1d93b35dc1c88..926e5249347f8 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -286,7 +286,7 @@ Numbers Convert a Python integer to a C :c:type:`unsigned long long` without overflow checking. -``n`` (:class:`int`) [Py_ssize_t] +``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a Python integer to a C :c:type:`Py_ssize_t`. ``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char] @@ -613,7 +613,7 @@ Building values ``K`` (:class:`int`) [unsigned long long] Convert a C :c:type:`unsigned long long` to a Python integer object. - ``n`` (:class:`int`) [Py_ssize_t] + ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a C :c:type:`Py_ssize_t` to a Python integer. ``c`` (:class:`bytes` of length 1) [char] diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47f0422e6100..7617487a462d3 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -84,8 +84,8 @@ called with a non-bytes parameter. | :attr:`%lu` | unsigned long | Equivalent to | | | | ``printf("%lu")``. [1]_ | +-------------------+---------------+--------------------------------+ - | :attr:`%zd` | Py_ssize_t | Equivalent to | - | | | ``printf("%zd")``. [1]_ | + | :attr:`%zd` | :c:type:`\ | Equivalent to | + | | Py_ssize_t` | ``printf("%zd")``. [1]_ | +-------------------+---------------+--------------------------------+ | :attr:`%zu` | size_t | Equivalent to | | | | ``printf("%zu")``. [1]_ | diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 7bfeca5958cc4..02ec1da1c3400 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -909,11 +909,11 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | +| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | +| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ @@ -959,7 +959,7 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ @@ -971,7 +971,7 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | +| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | +-----------------------------------------+---------------------------------+----------+ @@ -981,7 +981,7 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | +| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ @@ -1046,7 +1046,7 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: +-------------------------------------+----------+ | :c:data:`PyExc_IOError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | \(3) | +| :c:data:`PyExc_WindowsError` | [2]_ | +-------------------------------------+----------+ .. versionchanged:: 3.3 @@ -1054,10 +1054,10 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: Notes: -(1) +.. [1] This is a base class for other standard exceptions. -(2) +.. [2] Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. @@ -1087,7 +1087,7 @@ the variables: +------------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | +| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | +------------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | +------------------------------------------+---------------------------------+----------+ @@ -1115,5 +1115,5 @@ the variables: Notes: -(1) +.. [3] This is a base class for other standard warning categories. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 728df2100336e..41826c8a5e53c 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -97,7 +97,7 @@ PyWideStringList If *index* is greater than or equal to *list* length, append *item* to *list*. - *index* must be greater than or equal to 0. + *index* must be greater than or equal to ``0``. Python must be preinitialized to call this function. @@ -256,8 +256,8 @@ PyPreConfig Set the LC_CTYPE locale to the user preferred locale? - If equals to 0, set :c:member:`~PyPreConfig.coerce_c_locale` and - :c:member:`~PyPreConfig.coerce_c_locale_warn` members to 0. + If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and + :c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``. See the :term:`locale encoding`. @@ -265,9 +265,9 @@ PyPreConfig .. c:member:: int coerce_c_locale - If equals to 2, coerce the C locale. + If equals to ``2``, coerce the C locale. - If equals to 1, read the LC_CTYPE locale to decide if it should be + If equals to ``1``, read the LC_CTYPE locale to decide if it should be coerced. See the :term:`locale encoding`. @@ -282,8 +282,8 @@ PyPreConfig .. c:member:: int dev_mode - If non-zero, enables the :ref:`Python Development Mode `: - see :c:member:`PyConfig.dev_mode`. + :ref:`Python Development Mode `: see + :c:member:`PyConfig.dev_mode`. Default: ``-1`` in Python mode, ``0`` in isolated mode. @@ -329,8 +329,10 @@ PyPreConfig If non-zero, enable the :ref:`Python UTF-8 Mode `. - Set by the :option:`-X utf8 <-X>` command line option and the - :envvar:`PYTHONUTF8` environment variable. + Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option + and the :envvar:`PYTHONUTF8` environment variable. + + Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``. Default: ``-1`` in Python config and ``0`` in isolated config. @@ -555,7 +557,7 @@ PyConfig * Otherwise (``python -c code`` and ``python``), prepend an empty string, which means the current working directory. - Set to 1 by the :option:`-P` command line option and the + Set to ``1`` by the :option:`-P` command line option and the :envvar:`PYTHONSAFEPATH` environment variable. Default: ``0`` in Python config, ``1`` in isolated config. @@ -592,10 +594,10 @@ PyConfig .. c:member:: int buffered_stdio - If equals to 0 and :c:member:`~PyConfig.configure_c_stdio` is non-zero, + If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero, disable buffering on the C streams stdout and stderr. - Set to 0 by the :option:`-u` command line option and the + Set to ``0`` by the :option:`-u` command line option and the :envvar:`PYTHONUNBUFFERED` environment variable. stdin is always opened in buffered mode. @@ -604,11 +606,11 @@ PyConfig .. c:member:: int bytes_warning - If equals to 1, issue a warning when comparing :class:`bytes` or + If equals to ``1``, issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with :class:`int`. - If equal or greater to 2, raise a :exc:`BytesWarning` exception in these + If equal or greater to ``2``, raise a :exc:`BytesWarning` exception in these cases. Incremented by the :option:`-b` command line option. @@ -671,6 +673,9 @@ PyConfig If non-zero, enable the :ref:`Python Development Mode `. + Set to ``1`` by the :option:`-X dev <-X>` option and the + :envvar:`PYTHONDEVMODE` environment variable. + Default: ``-1`` in Python mode, ``0`` in isolated mode. .. c:member:: int dump_refs @@ -800,7 +805,7 @@ PyConfig Enter interactive mode after executing a script or a command. - If greater than 0, enable inspect: when a script is passed as first + If greater than ``0``, enable inspect: when a script is passed as first argument or the -c option is used, enter interactive mode after executing the script or the command, even when :data:`sys.stdin` does not appear to be a terminal. @@ -818,7 +823,7 @@ PyConfig .. c:member:: int interactive - If greater than 0, enable the interactive mode (REPL). + If greater than ``0``, enable the interactive mode (REPL). Incremented by the :option:`-i` command line option. @@ -826,17 +831,19 @@ PyConfig .. c:member:: int isolated - If greater than 0, enable isolated mode: + If greater than ``0``, enable isolated mode: - * Set :c:member:`~PyConfig.safe_path` to 1: + * Set :c:member:`~PyConfig.safe_path` to ``1``: don't prepend a potentially unsafe path to :data:`sys.path` at Python startup. - * Set :c:member:`~PyConfig.use_environment` to 0. - * Set :c:member:`~PyConfig.user_site_directory` to 0: don't add the user + * Set :c:member:`~PyConfig.use_environment` to ``0``. + * Set :c:member:`~PyConfig.user_site_directory` to ``0``: don't add the user site directory to :data:`sys.path`. * Python REPL doesn't import :mod:`readline` nor enable default readline configuration on interactive prompts. + Set to ``1`` by the :option:`-I` command line option. + Default: ``0`` in Python mode, ``1`` in isolated mode. See also :c:member:`PyPreConfig.isolated`. @@ -906,7 +913,7 @@ PyConfig Module search paths: :data:`sys.path`. - If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, + If :c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, :c:func:`Py_InitializeFromConfig` will replace :c:member:`~PyConfig.module_search_paths` and sets :c:member:`~PyConfig.module_search_paths_set` to ``1``. @@ -970,7 +977,7 @@ PyConfig .. c:member:: int parser_debug - Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending + Parser debug mode. If greater than ``0``, turn on parser debugging output (for expert only, depending on compilation options). Incremented by the :option:`-d` command line option. Set to the @@ -981,7 +988,7 @@ PyConfig .. c:member:: int pathconfig_warnings If non-zero, calculation of path configuration is allowed to log - warnings into ``stderr``. If equals to 0, suppress these warnings. + warnings into ``stderr``. If equals to ``0``, suppress these warnings. Default: ``1`` in Python mode, ``0`` in isolated mode. @@ -1031,7 +1038,7 @@ PyConfig .. c:member:: int quiet - Quiet mode. If greater than 0, don't display the copyright and version at + Quiet mode. If greater than ``0``, don't display the copyright and version at Python startup in interactive mode. Incremented by the :option:`-q` command line option. @@ -1071,7 +1078,7 @@ PyConfig Show total reference count at exit? - Set to 1 by :option:`-X showrefcount <-X>` command line option. + Set to ``1`` by :option:`-X showrefcount <-X>` command line option. Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` macro must be defined). @@ -1150,6 +1157,8 @@ PyConfig If equals to zero, ignore the :ref:`environment variables `. + Set to ``0`` by the :option:`-E` environment variable. + Default: ``1`` in Python config and ``0`` in isolated config. .. c:member:: int user_site_directory @@ -1164,11 +1173,11 @@ PyConfig .. c:member:: int verbose - Verbose mode. If greater than 0, print a message each time a module is + Verbose mode. If greater than ``0``, print a message each time a module is imported, showing the place (filename or built-in module) from which it is loaded. - If greater or equal to 2, print a message for each file that is checked + If greater or equal to ``2``, print a message for each file that is checked for when searching for a module. Also provides information on module cleanup at exit. @@ -1199,7 +1208,7 @@ PyConfig .. c:member:: int write_bytecode - If equal to 0, Python won't try to write ``.pyc`` files on the import of + If equal to ``0``, Python won't try to write ``.pyc`` files on the import of source modules. Set to ``0`` by the :option:`-B` command line option and the @@ -1280,7 +1289,11 @@ Example setting the program name:: } More complete example modifying the default configuration, read the -configuration, and then override some parameters:: +configuration, and then override some parameters. Note that since +3.11, many parameters are not calculated until initialization, and +so values cannot be read from the configuration structure. Any values +set before initialize is called will be left unchanged by +initialization:: PyStatus init_python(const char *program_name) { @@ -1305,7 +1318,15 @@ configuration, and then override some parameters:: goto done; } - /* Append our custom search path to sys.path */ + /* Specify sys.path explicitly */ + /* If you want to modify the default set of paths, finish + initialization first and then use PySys_GetObject("path") */ + config.module_search_paths_set = 1; + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/stdlib"); + if (PyStatus_Exception(status)) { + goto done; + } status = PyWideStringList_Append(&config.module_search_paths, L"/path/to/more/modules"); if (PyStatus_Exception(status)) { @@ -1400,18 +1421,18 @@ Python Path Configuration If at least one "output field" is not set, Python calculates the path configuration to fill unset fields. If -:c:member:`~PyConfig.module_search_paths_set` is equal to 0, +:c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, :c:member:`~PyConfig.module_search_paths` is overridden and -:c:member:`~PyConfig.module_search_paths_set` is set to 1. +:c:member:`~PyConfig.module_search_paths_set` is set to ``1``. It is possible to completely ignore the function calculating the default path configuration by setting explicitly all path configuration output fields listed above. A string is considered as set even if it is non-empty. ``module_search_paths`` is considered as set if -``module_search_paths_set`` is set to 1. In this case, path -configuration input fields are ignored as well. +``module_search_paths_set`` is set to ``1``. In this case, +``module_search_paths`` will be used without modification. -Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when +Set :c:member:`~PyConfig.pathconfig_warnings` to ``0`` to suppress warnings when calculating the path configuration (Unix only, Windows does not log any warning). If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` @@ -1445,10 +1466,10 @@ The following configuration files are used by the path configuration: If a ``._pth`` file is present: -* Set :c:member:`~PyConfig.isolated` to 1. -* Set :c:member:`~PyConfig.use_environment` to 0. -* Set :c:member:`~PyConfig.site_import` to 0. -* Set :c:member:`~PyConfig.safe_path` to 1. +* Set :c:member:`~PyConfig.isolated` to ``1``. +* Set :c:member:`~PyConfig.use_environment` to ``0``. +* Set :c:member:`~PyConfig.site_import` to ``0``. +* Set :c:member:`~PyConfig.safe_path` to ``1``. The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable` @@ -1511,7 +1532,7 @@ initialization, the core feature of :pep:`432`: Private provisional API: -* :c:member:`PyConfig._init_main`: if set to 0, +* :c:member:`PyConfig._init_main`: if set to ``0``, :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. * :c:member:`PyConfig._isolated_interpreter`: if non-zero, disallow threads, subprocesses and fork. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 9efac0b83d024..e53c826c58d13 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -537,6 +537,13 @@ data attributes of a new object type, and another is used to describe the value of a complex number. These will be discussed together with the functions that use them. +.. c:type:: Py_ssize_t + + A signed integral type such that ``sizeof(Py_ssize_t) == sizeof(size_t)``. + C99 doesn't define such a thing directly (size_t is an unsigned integral type). + See :pep:`353` for details. ``PY_SSIZE_T_MAX`` is the largest positive value + of type :c:type:`Py_ssize_t`. + .. _api-exceptions: diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 11c9c67d36a67..70b91f8c2d0ca 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -273,11 +273,11 @@ Number Protocol .. c:function:: Py_ssize_t PyNumber_AsSsize_t(PyObject *o, PyObject *exc) - Returns *o* converted to a Py_ssize_t value if *o* can be interpreted as an + Returns *o* converted to a :c:type:`Py_ssize_t` value if *o* can be interpreted as an integer. If the call fails, an exception is raised and ``-1`` is returned. If *o* can be converted to a Python int but the attempt to - convert to a Py_ssize_t value would raise an :exc:`OverflowError`, then the + convert to a :c:type:`Py_ssize_t` value would raise an :exc:`OverflowError`, then the *exc* argument is the type of exception that will be raised (usually :exc:`IndexError` or :exc:`OverflowError`). If *exc* is ``NULL``, then the exception is cleared and the value is clipped to ``PY_SSIZE_T_MIN`` for a negative diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 9dcfd769c64a0..07a625bac02fc 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -258,7 +258,7 @@ Object Protocol .. versionchanged:: 3.2 The return type is now Py_hash_t. This is a signed integer the same size - as Py_ssize_t. + as :c:type:`Py_ssize_t`. .. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 391907c8c2976..738bd77e9ce42 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -109,11 +109,17 @@ objects. It is a good idea to use this macro whenever decrementing the reference count of an object that might be traversed during garbage collection. +.. c:function:: void Py_IncRef(PyObject *o) + + Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + It can be used for runtime dynamic embedding of Python. + + +.. c:function:: void Py_DecRef(PyObject *o) + + Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + It can be used for runtime dynamic embedding of Python. -The following functions are for runtime dynamic embedding of Python: -``Py_IncRef(PyObject *o)``, ``Py_DecRef(PyObject *o)``. They are -simply exported function versions of :c:func:`Py_XINCREF` and -:c:func:`Py_XDECREF`, respectively. The following functions or macros are only for use within the interpreter core: :c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 5e8d993100f63..7b714678444be 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -348,7 +348,7 @@ accessible to C code. They all work with the current interpreter thread's leaks.) Note that ``#`` format characters should always be treated as - ``Py_ssize_t``, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. + :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. :func:`sys.audit` performs the same function from Python code. @@ -356,7 +356,7 @@ accessible to C code. They all work with the current interpreter thread's .. versionchanged:: 3.8.2 - Require ``Py_ssize_t`` for ``#`` format characters. Previously, an + Require :c:type:`Py_ssize_t` for ``#`` format characters. Previously, an unavoidable deprecation warning was raised. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ed434d8fd44fb..b3f371bb9c062 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -43,13 +43,13 @@ Quick Reference +================================================+===================================+===================+===+===+===+===+ | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | + | :c:member:`~PyTypeObject.tp_basicsize` | :c:type:`Py_ssize_t` | | X | X | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | + | :c:member:`~PyTypeObject.tp_itemsize` | :c:type:`Py_ssize_t` | | | X | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_vectorcall_offset` | Py_ssize_t | | | X | | X | + | :c:member:`~PyTypeObject.tp_vectorcall_offset` | :c:type:`Py_ssize_t` | | | X | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | | | | __getattr__ | | | | | @@ -96,7 +96,7 @@ Quick Reference | | | __gt__, | | | | | | | | __ge__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | + | :c:member:`~PyTypeObject.tp_weaklistoffset` | :c:type:`Py_ssize_t` | | | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -117,7 +117,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | | | | __delete__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | + | :c:member:`~PyTypeObject.tp_dictoffset` | :c:type:`Py_ssize_t` | | | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -333,7 +333,7 @@ slot typedefs | :c:type:`allocfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyTypeObject` * | | -| | Py_ssize_t | | +| | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ | :c:type:`destructor` | void * | void | +-----------------------------+-----------------------------+----------------------+ @@ -405,7 +405,7 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`iternextfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`lenfunc` | :c:type:`PyObject` * | Py_ssize_t | +| :c:type:`lenfunc` | :c:type:`PyObject` * | :c:type:`Py_ssize_t` | +-----------------------------+-----------------------------+----------------------+ | :c:type:`getbufferproc` | .. line-block:: | int | | | | | @@ -438,12 +438,12 @@ slot typedefs | :c:type:`ssizeargfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyObject` * | | -| | Py_ssize_t | | +| | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ | :c:type:`ssizeobjargproc` | .. line-block:: | int | | | | | | | :c:type:`PyObject` * | | -| | Py_ssize_t | | +| | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ | :c:type:`objobjproc` | .. line-block:: | int | | | | | diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index d139112578ca9..eb99013395360 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -149,7 +149,7 @@ access to internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` is deprecated. -.. c:function:: unsigned int PyUnicode_KIND(PyObject *o) +.. c:function:: int PyUnicode_KIND(PyObject *o) Return one of the PyUnicode kind constants (see above) that indicate how many bytes per character this Unicode object uses to store its data. *o* has to @@ -168,7 +168,7 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: void PyUnicode_WRITE(unsigned int kind, void *data, \ +.. c:function:: void PyUnicode_WRITE(int kind, void *data, \ Py_ssize_t index, Py_UCS4 value) Write into a canonical representation *data* (as obtained with @@ -181,7 +181,7 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_READ(unsigned int kind, void *data, \ +.. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, \ Py_ssize_t index) Read a code point from a canonical representation *data* (as obtained with @@ -497,11 +497,11 @@ APIs: | :attr:`%llu` | unsigned long long | Equivalent to | | | | ``printf("%llu")``. [1]_ | +-------------------+---------------------+----------------------------------+ - | :attr:`%zd` | Py_ssize_t | Equivalent to | - | | | ``printf("%zd")``. [1]_ | + | :attr:`%zd` | :c:type:`\ | Equivalent to | + | | Py_ssize_t` | ``printf("%zd")``. [1]_ | +-------------------+---------------------+----------------------------------+ - | :attr:`%zi` | Py_ssize_t | Equivalent to | - | | | ``printf("%zi")``. [1]_ | + | :attr:`%zi` | :c:type:`\ | Equivalent to | + | | Py_ssize_t` | ``printf("%zi")``. [1]_ | +-------------------+---------------------+----------------------------------+ | :attr:`%zu` | size_t | Equivalent to | | | | ``printf("%zu")``. [1]_ | diff --git a/Doc/data/python3.11.abi b/Doc/data/python3.11.abi new file mode 100644 index 0000000000000..c5a1fdaea6fff --- /dev/null +++ b/Doc/data/python3.11.abi @@ -0,0 +1,16595 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index c7b92c6ea24ca..28d0350f6f114 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -106,8 +106,7 @@ Using DLLs in Practice Windows Python is built in Microsoft Visual C++; using other compilers may or -may not work (though Borland seems to). The rest of this section is MSVC++ -specific. +may not work. The rest of this section is MSVC++ specific. When creating DLLs in Windows, you must pass :file:`pythonXY.lib` to the linker. To build two DLLs, spam and ni (which uses C functions found in spam), you could @@ -134,4 +133,3 @@ Developer Studio will throw in a lot of import libraries that you do not really need, adding about 100K to your executable. To get rid of them, use the Project Settings dialog, Link tab, to specify *ignore default libraries*. Add the correct :file:`msvcrtxx.lib` to the list of libraries. - diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ff83a1b8134b7..a624fdb07a17d 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -324,8 +324,7 @@ Can Python be compiled to machine code, C or some other language? `Cython `_ compiles a modified version of Python with optional annotations into C extensions. `Nuitka `_ is an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC `_. +Python language. How does Python manage memory? diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index 6b95819c8ee85..df7ab71f4b662 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -26,8 +26,8 @@ with running programs from the Windows command line then everything will seem obvious; otherwise, you might need a little more guidance. Unless you use some sort of integrated development environment, you will end up -*typing* Windows commands into what is variously referred to as a "DOS window" -or "Command prompt window". Usually you can create such a window from your +*typing* Windows commands into what is referred to as a +"Command prompt window". Usually you can create such a window from your search bar by searching for ``cmd``. You should be able to recognize when you have started such a window because you will see a Windows "command prompt", which usually looks like this: @@ -186,9 +186,6 @@ Embedding the Python interpreter in a Windows app can be summarized as follows: by the Windows ``GetProcAddress()`` routine. Macros can make using these pointers transparent to any C code that calls routines in Python's C API. - Borland note: convert :file:`python{NN}.lib` to OMF format using Coff2Omf.exe - first. - .. XXX what about static linking? 2. If you use SWIG, it is easy to create a Python "extension module" that will @@ -279,4 +276,3 @@ How do I check for a keypress without blocking? Use the :mod:`msvcrt` module. This is a standard Windows-specific extension module. It defines a function ``kbhit()`` which checks whether a keyboard hit is present, and ``getch()`` which gets one character without echoing it. - diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 75db433abd600..27e06c9ffcf4e 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1125,7 +1125,16 @@ Glossary See also :term:`borrowed reference`. text encoding - A codec which encodes Unicode strings to bytes. + A string in Python is a sequence of Unicode code points (in range + ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be + serialized as a sequence of bytes. + + Serializing a string into a sequence of bytes is known as "encoding", and + recreating the string from the sequence of bytes is known as "decoding". + + There are a variety of different text serialization + :ref:`codecs `, which are collectively referred to as + "text encodings". text file A :term:`file object` able to read and write :class:`str` objects. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 04b1a2cac0b04..72fb3208c3752 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1347,7 +1347,7 @@ Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c`` /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type ``Py_ssize_t``, and will +declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will be parsed by the ``'O&'`` format unit, which will call the ``ssize_t_converter`` converter function. ``ssize_t`` variables automatically support default values. diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 4d76c27332ccd..93400627136e3 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -178,10 +178,11 @@ following example:: raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, ...) -The call to :func:`basicConfig` should come *before* any calls to :func:`debug`, -:func:`info` etc. As it's intended as a one-off simple configuration facility, -only the first call will actually do anything: subsequent calls are effectively -no-ops. +The call to :func:`basicConfig` should come *before* any calls to +:func:`debug`, :func:`info`, etc. Otherwise, those functions will call +:func:`basicConfig` for you with the default options. As it's intended as a +one-off simple configuration facility, only the first call will actually do +anything: subsequent calls are effectively no-ops. If you run the above script several times, the messages from successive runs are appended to the file *example.log*. If you want each run to start afresh, diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 24bbd90d02cf7..8bd23daee7397 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,12 +90,20 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | 3.11 | :pep:`563`: | +| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. +.. [1] + ``from __future__ import annotations`` was previously scheduled to + become mandatory in Python 3.10, but the Python Steering Council + twice decided to delay the change + (`announcement for Python 3.10 `__; + `announcement for Python 3.11 `__). + No final decision has been made yet. See also :pep:`563` and :pep:`649`. + .. seealso:: diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index edb4bf86e5a0a..9f20a30193fa7 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -13,8 +13,9 @@ single: AIFF-C -.. deprecated:: 3.11 - The :mod:`aifc` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`aifc` module is deprecated + (see :pep:`PEP 594 <594#aifc>` for details). -------------- diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index b5a2b794c2385..76efed95bd4cc 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -555,7 +555,7 @@ disallowed. fromfile_prefix_chars ^^^^^^^^^^^^^^^^^^^^^ -Sometimes, when dealing with a particularly long argument lists, it +Sometimes, when dealing with a particularly long argument list, it may make sense to keep the list of arguments in a file rather than typing it out at the command line. If the ``fromfile_prefix_chars=`` argument is given to the :class:`ArgumentParser` constructor, then arguments that start with any of the @@ -1698,7 +1698,7 @@ Sub-commands .. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ [parser_class], [action], \ - [option_string], [dest], [required], \ + [option_strings], [dest], [required], \ [help], [metavar]) Many programs split up their functionality into a number of sub-commands, diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst index 4354444a1d331..4eb6a79d4dfbf 100644 --- a/Doc/library/asynchat.rst +++ b/Doc/library/asynchat.rst @@ -10,8 +10,9 @@ **Source code:** :source:`Lib/asynchat.py` -.. deprecated:: 3.6 - :mod:`asynchat` will be removed in Python 3.12 (:pep:`594`). +.. deprecated-removed:: 3.6 3.12 + The :mod:`asynchat` module is deprecated + (see :pep:`PEP 594 <594#asynchat>` for details). Please use :mod:`asyncio` instead. -------------- diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 72355d356f205..97431d103cf4b 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -311,7 +311,7 @@ StreamWriter handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). - .. versionadded:: 3.8 + .. versionadded:: 3.11 .. method:: is_closing() diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index e5000532a895d..28d0b21e8180b 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -124,6 +124,7 @@ Constants ========= .. data:: asyncio.subprocess.PIPE + :module: Can be passed to the *stdin*, *stdout* or *stderr* parameters. @@ -137,11 +138,13 @@ Constants attributes will point to :class:`StreamReader` instances. .. data:: asyncio.subprocess.STDOUT + :module: Special value that can be used as the *stderr* argument and indicates that standard error should be redirected into standard output. .. data:: asyncio.subprocess.DEVNULL + :module: Special value that can be used as the *stdin*, *stdout* or *stderr* argument to process creation functions. It indicates that the special file @@ -157,6 +160,7 @@ wrapper that allows communicating with subprocesses and watching for their completion. .. class:: asyncio.subprocess.Process + :module: An object that wraps OS processes created by the :func:`create_subprocess_exec` and :func:`create_subprocess_shell` diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 8f4d55a9de059..66c7c4c24a918 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -17,7 +17,6 @@ await asyncio.sleep(1) print('... World!') - # Python 3.7+ asyncio.run(main()) asyncio is a library to write **concurrent** code using diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst index e481e13db76f7..0084d754419d0 100644 --- a/Doc/library/asyncore.rst +++ b/Doc/library/asyncore.rst @@ -13,8 +13,9 @@ **Source code:** :source:`Lib/asyncore.py` -.. deprecated:: 3.6 - :mod:`asyncore` will be removed in Python 3.12 (:pep:`594`). +.. deprecated-removed:: 3.6 3.12 + The :mod:`asyncore` module is deprecated + (see :pep:`PEP 594 <594#asyncore>` for details). Please use :mod:`asyncio` instead. -------------- diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst index eae206084f090..1f96575d08f5b 100644 --- a/Doc/library/audioop.rst +++ b/Doc/library/audioop.rst @@ -5,8 +5,9 @@ :synopsis: Manipulate raw audio data. :deprecated: -.. deprecated:: 3.11 - The :mod:`audioop` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`audioop` module is deprecated + (see :pep:`PEP 594 <594#audioop>` for details). -------------- diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 19efc2df9483d..5a0815faa38ea 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -49,7 +49,7 @@ The :mod:`binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, strict_mode=False) +.. function:: a2b_base64(string, /, *, strict_mode=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. @@ -121,8 +121,6 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.0 The result is always unsigned. - To generate the same numeric value when using Python 2 or earlier, - use ``crc32(data) & 0xffffffff``. .. function:: b2a_hex(data[, sep[, bytes_per_sep=1]]) hexlify(data[, sep[, bytes_per_sep=1]]) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index edcd4aeb24aa7..513675d3685a5 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -35,8 +35,11 @@ The following functions are provided: ``all(val >= x for val in a[i : hi])`` for the right side. *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). + extract a comparison key from each element in the array. To support + searching complex records, the key function is not applied to the *x* value. + + If *key* is ``None``, the elements are compared directly with no + intervening function call. .. versionchanged:: 3.10 Added the *key* parameter. @@ -53,8 +56,11 @@ The following functions are provided: ``all(val > x for val in a[i : hi])`` for the right side. *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). + extract a comparison key from each element in the array. To support + searching complex records, the key function is not applied to the *x* value. + + If *key* is ``None``, the elements are compared directly with no + intervening function call. .. versionchanged:: 3.10 Added the *key* parameter. @@ -64,14 +70,13 @@ The following functions are provided: Insert *x* in *a* in sorted order. - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). - This function first runs :func:`bisect_left` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. + To support inserting records in a table, the *key* function (if any) is + applied to *x* for the search step but not for the insertion step. + Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) insertion step. @@ -85,14 +90,13 @@ The following functions are provided: Similar to :func:`insort_left`, but inserting *x* in *a* after any existing entries of *x*. - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). - This function first runs :func:`bisect_right` to locate an insertion point. Next, it runs the :meth:`insert` method on *a* to insert *x* at the appropriate position to maintain sort order. + To support inserting records in a table, the *key* function (if any) is + applied to *x* for the search step but not for the insertion step. + Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) insertion step. @@ -194,8 +198,42 @@ a 'B', and so on:: >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] -One technique to avoid repeated calls to a key function is to search a list of -precomputed keys to find the index of a record:: +The :func:`bisect` and :func:`insort` functions also work with lists of +tuples. The *key* argument can serve to extract the field used for ordering +records in a table:: + + >>> from collections import namedtuple + >>> from operator import attrgetter + >>> from bisect import bisect, insort + >>> from pprint import pprint + + >>> Movie = namedtuple('Movie', ('name', 'released', 'director')) + + >>> movies = [ + ... Movie('Jaws', 1975, 'Speilberg'), + ... Movie('Titanic', 1997, 'Cameron'), + ... Movie('The Birds', 1963, 'Hitchcock'), + ... Movie('Aliens', 1986, 'Scott') + ... ] + + >>> # Find the first movie released on or after 1960 + >>> by_year = attrgetter('released') + >>> movies.sort(key=by_year) + >>> movies[bisect(movies, 1960, key=by_year)] + Movie(name='The Birds', released=1963, director='Hitchcock') + + >>> # Insert a movie while maintaining sort order + >>> romance = Movie('Love Story', 1970, 'Hiller') + >>> insort(movies, romance, key=by_year) + >>> pprint(movies) + [Movie(name='The Birds', released=1963, director='Hitchcock'), + Movie(name='Love Story', released=1970, director='Hiller'), + Movie(name='Jaws', released=1975, director='Speilberg'), + Movie(name='Aliens', released=1986, director='Scott'), + Movie(name='Titanic', released=1997, director='Cameron')] + +If the key function is expensive, it is possible to avoid repeated function +calls by searching a list of precomputed keys to find the index of a record:: >>> data = [('red', 5), ('blue', 1), ('yellow', 8), ('black', 0)] >>> data.sort(key=lambda r: r[1]) # Or use operator.itemgetter(1). @@ -208,4 +246,3 @@ precomputed keys to find the index of a record:: ('red', 5) >>> data[bisect_left(keys, 8)] ('yellow', 8) - diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index 7e697408af07b..5976c90029c22 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -15,8 +15,9 @@ single: URL single: Common Gateway Interface -.. deprecated:: 3.11 - The :mod:`cgi` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`cgi` module is deprecated + (see :pep:`PEP 594 <594#cgi>` for details and alternatives). -------------- diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst index 349414610bd40..7f00bcd55c1e5 100644 --- a/Doc/library/cgitb.rst +++ b/Doc/library/cgitb.rst @@ -16,8 +16,9 @@ single: exceptions; in CGI scripts single: tracebacks; in CGI scripts -.. deprecated:: 3.11 - The :mod:`cgitb` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`cgitb` module is deprecated + (see :pep:`PEP 594 <594#cgitb>` for details). -------------- diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst index 7999420f536d7..3b88e55b14788 100644 --- a/Doc/library/chunk.rst +++ b/Doc/library/chunk.rst @@ -17,8 +17,9 @@ single: Real Media File Format single: RMFF -.. deprecated:: 3.11 - The :mod:`chunk` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`chunk` module is deprecated + (see :pep:`PEP 594 <594#chunk>` for details). -------------- diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 26123fd0d0060..d131408175fd1 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -23,11 +23,11 @@ This module defines base classes for standard Python codecs (encoders and decoders) and provides access to the internal Python codec registry, which manages the codec and error handling lookup process. Most standard codecs -are :term:`text encodings `, which encode text to bytes, -but there are also codecs provided that encode text to text, and bytes to -bytes. Custom codecs may encode and decode between arbitrary types, but some -module features are restricted to use specifically with -:term:`text encodings `, or with codecs that encode to +are :term:`text encodings `, which encode text to bytes (and +decode bytes to text), but there are also codecs provided that encode text to +text, and bytes to bytes. Custom codecs may encode and decode between arbitrary +types, but some module features are restricted to be used specifically with +:term:`text encodings ` or with codecs that encode to :class:`bytes`. The module defines the following functions for encoding and decoding with @@ -300,58 +300,56 @@ codec will handle encoding and decoding errors. Error Handlers ^^^^^^^^^^^^^^ -To simplify and standardize error handling, -codecs may implement different error handling schemes by -accepting the *errors* string argument. The following string values are -defined and implemented by all standard Python codecs: +To simplify and standardize error handling, codecs may implement different +error handling schemes by accepting the *errors* string argument: -.. tabularcolumns:: |l|L| - -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass); | -| | this is the default. Implemented in | -| | :func:`strict_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'ignore'`` | Ignore the malformed data and continue | -| | without further notice. Implemented in | -| | :func:`ignore_errors`. | -+-------------------------+-----------------------------------------------+ - -The following error handlers are only applicable to -:term:`text encodings `: + >>> 'German ß, ♬'.encode(encoding='ascii', errors='backslashreplace') + b'German \\xdf, \\u266c' + >>> 'German ß, ♬'.encode(encoding='ascii', errors='xmlcharrefreplace') + b'German ß, ♬' .. index:: + pair: strict; error handler's name + pair: ignore; error handler's name + pair: replace; error handler's name + pair: backslashreplace; error handler's name + pair: surrogateescape; error handler's name single: ? (question mark); replacement character single: \ (backslash); escape sequence single: \x; escape sequence single: \u; escape sequence single: \U; escape sequence - single: \N; escape sequence + +The following error handlers can be used with all Python +:ref:`standard-encodings` codecs: + +.. tabularcolumns:: |l|L| +-------------------------+-----------------------------------------------+ | Value | Meaning | +=========================+===============================================+ -| ``'replace'`` | Replace with a suitable replacement | -| | marker; Python will use the official | -| | ``U+FFFD`` REPLACEMENT CHARACTER for the | -| | built-in codecs on decoding, and '?' on | -| | encoding. Implemented in | -| | :func:`replace_errors`. | +| ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass), | +| | this is the default. Implemented in | +| | :func:`strict_errors`. | +-------------------------+-----------------------------------------------+ -| ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | -| | reference (only for encoding). Implemented | -| | in :func:`xmlcharrefreplace_errors`. | +| ``'ignore'`` | Ignore the malformed data and continue without| +| | further notice. Implemented in | +| | :func:`ignore_errors`. | ++-------------------------+-----------------------------------------------+ +| ``'replace'`` | Replace with a replacement marker. On | +| | encoding, use ``?`` (ASCII character). On | +| | decoding, use ``�`` (U+FFFD, the official | +| | REPLACEMENT CHARACTER). Implemented in | +| | :func:`replace_errors`. | +-------------------------+-----------------------------------------------+ | ``'backslashreplace'`` | Replace with backslashed escape sequences. | +| | On encoding, use hexadecimal form of Unicode | +| | code point with formats ``\xhh`` ``\uxxxx`` | +| | ``\Uxxxxxxxx``. On decoding, use hexadecimal | +| | form of byte value with format ``\xhh``. | | | Implemented in | | | :func:`backslashreplace_errors`. | +-------------------------+-----------------------------------------------+ -| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences | -| | (only for encoding). Implemented in | -| | :func:`namereplace_errors`. | -+-------------------------+-----------------------------------------------+ | ``'surrogateescape'`` | On decoding, replace byte with individual | | | surrogate code ranging from ``U+DC80`` to | | | ``U+DCFF``. This code will then be turned | @@ -361,27 +359,55 @@ The following error handlers are only applicable to | | more.) | +-------------------------+-----------------------------------------------+ +.. index:: + pair: xmlcharrefreplace; error handler's name + pair: namereplace; error handler's name + single: \N; escape sequence + +The following error handlers are only applicable to encoding (within +:term:`text encodings `): + ++-------------------------+-----------------------------------------------+ +| Value | Meaning | ++=========================+===============================================+ +| ``'xmlcharrefreplace'`` | Replace with XML/HTML numeric character | +| | reference, which is a decimal form of Unicode | +| | code point with format ``&#num;`` Implemented | +| | in :func:`xmlcharrefreplace_errors`. | ++-------------------------+-----------------------------------------------+ +| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences, | +| | what appears in the braces is the Name | +| | property from Unicode Character Database. | +| | Implemented in :func:`namereplace_errors`. | ++-------------------------+-----------------------------------------------+ + +.. index:: + pair: surrogatepass; error handler's name + In addition, the following error handler is specific to the given codecs: +-------------------+------------------------+-------------------------------------------+ | Value | Codecs | Meaning | +===================+========================+===========================================+ -|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | -| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | -| | utf-32-be, utf-32-le | presence of surrogates as an error. | +|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding surrogate code| +| | utf-16-be, utf-16-le, | point (``U+D800`` - ``U+DFFF``) as normal | +| | utf-32-be, utf-32-le | code point. Otherwise these codecs treat | +| | | the presence of surrogate code point in | +| | | :class:`str` as an error. | +-------------------+------------------------+-------------------------------------------+ .. versionadded:: 3.1 The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. .. versionchanged:: 3.4 - The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs. + The ``'surrogatepass'`` error handler now works with utf-16\* and utf-32\* + codecs. .. versionadded:: 3.5 The ``'namereplace'`` error handler. .. versionchanged:: 3.5 - The ``'backslashreplace'`` error handlers now works with decoding and + The ``'backslashreplace'`` error handler now works with decoding and translating. The set of allowed values can be extended by registering a new named error @@ -424,42 +450,59 @@ functions: .. function:: strict_errors(exception) - Implements the ``'strict'`` error handling: each encoding or - decoding error raises a :exc:`UnicodeError`. + Implements the ``'strict'`` error handling. + Each encoding or decoding error raises a :exc:`UnicodeError`. -.. function:: replace_errors(exception) - Implements the ``'replace'`` error handling (for :term:`text encodings - ` only): substitutes ``'?'`` for encoding errors - (to be encoded by the codec), and ``'\ufffd'`` (the Unicode replacement - character) for decoding errors. +.. function:: ignore_errors(exception) + Implements the ``'ignore'`` error handling. -.. function:: ignore_errors(exception) + Malformed data is ignored; encoding or decoding is continued without + further notice. - Implements the ``'ignore'`` error handling: malformed data is ignored and - encoding or decoding is continued without further notice. +.. function:: replace_errors(exception) -.. function:: xmlcharrefreplace_errors(exception) + Implements the ``'replace'`` error handling. - Implements the ``'xmlcharrefreplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by an appropriate XML character reference. + Substitutes ``?`` (ASCII character) for encoding errors or ``�`` (U+FFFD, + the official REPLACEMENT CHARACTER) for decoding errors. .. function:: backslashreplace_errors(exception) - Implements the ``'backslashreplace'`` error handling (for - :term:`text encodings ` only): malformed data is - replaced by a backslashed escape sequence. + Implements the ``'backslashreplace'`` error handling. + + Malformed data is replaced by a backslashed escape sequence. + On encoding, use the hexadecimal form of Unicode code point with formats + ``\xhh`` ``\uxxxx`` ``\Uxxxxxxxx``. On decoding, use the hexadecimal form of + byte value with format ``\xhh``. + + .. versionchanged:: 3.5 + Works with decoding and translating. + + +.. function:: xmlcharrefreplace_errors(exception) + + Implements the ``'xmlcharrefreplace'`` error handling (for encoding within + :term:`text encoding` only). + + The unencodable character is replaced by an appropriate XML/HTML numeric + character reference, which is a decimal form of Unicode code point with + format ``&#num;`` . + .. function:: namereplace_errors(exception) - Implements the ``'namereplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by a ``\N{...}`` escape sequence. + Implements the ``'namereplace'`` error handling (for encoding within + :term:`text encoding` only). + + The unencodable character is replaced by a ``\N{...}`` escape sequence. The + set of characters that appear in the braces is the Name property from + Unicode Character Database. For example, the German lowercase letter ``'ß'`` + will be converted to byte sequence ``\N{LATIN SMALL LETTER SHARP S}`` . .. versionadded:: 3.5 @@ -473,7 +516,7 @@ The base :class:`Codec` class defines these methods which also define the function interfaces of the stateless encoder and decoder: -.. method:: Codec.encode(input[, errors]) +.. method:: Codec.encode(input, errors='strict') Encodes the object *input* and returns a tuple (output object, length consumed). For instance, :term:`text encoding` converts @@ -491,7 +534,7 @@ function interfaces of the stateless encoder and decoder: of the output object type in this situation. -.. method:: Codec.decode(input[, errors]) +.. method:: Codec.decode(input, errors='strict') Decodes the object *input* and returns a tuple (output object, length consumed). For instance, for a :term:`text encoding`, decoding converts @@ -558,7 +601,7 @@ define in order to be compatible with the Python codec registry. object. - .. method:: encode(object[, final]) + .. method:: encode(object, final=False) Encodes *object* (taking the current state of the encoder into account) and returns the resulting encoded object. If this is the last call to @@ -615,7 +658,7 @@ define in order to be compatible with the Python codec registry. object. - .. method:: decode(object[, final]) + .. method:: decode(object, final=False) Decodes *object* (taking the current state of the decoder into account) and returns the resulting decoded object. If this is the last call to @@ -749,7 +792,7 @@ compatible with the Python codec registry. :func:`register_error`. - .. method:: read([size[, chars, [firstline]]]) + .. method:: read(size=-1, chars=-1, firstline=False) Decodes data from the stream and returns the resulting object. @@ -775,7 +818,7 @@ compatible with the Python codec registry. available on the stream, these should be read too. - .. method:: readline([size[, keepends]]) + .. method:: readline(size=None, keepends=True) Read one line from the input stream and return the decoded data. @@ -786,7 +829,7 @@ compatible with the Python codec registry. returned. - .. method:: readlines([sizehint[, keepends]]) + .. method:: readlines(sizehint=None, keepends=True) Read all lines available on the input stream and return them as a list of lines. @@ -877,7 +920,7 @@ Encodings and Unicode --------------------- Strings are stored internally as sequences of code points in -range ``0x0``--``0x10FFFF``. (See :pep:`393` for +range ``U+0000``--``U+10FFFF``. (See :pep:`393` for more details about the implementation.) Once a string object is used outside of CPU and memory, endianness and how these arrays are stored as bytes become an issue. As with other @@ -958,7 +1001,7 @@ encoding was used for encoding a string. Each charmap encoding can decode any random byte sequence. However that's not possible with UTF-8, as UTF-8 byte sequences have a structure that doesn't allow arbitrary byte sequences. To increase the reliability with which a UTF-8 encoding can be -detected, Microsoft invented a variant of UTF-8 (that Python 2.5 calls +detected, Microsoft invented a variant of UTF-8 (that Python calls ``"utf-8-sig"``) for its Notepad program: Before any of the Unicode characters is written to the file, a UTF-8 encoded BOM (which looks like this as a byte sequence: ``0xef``, ``0xbb``, ``0xbf``) is written. As it's rather improbable diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 3189ece048a26..efba4236bcbcc 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -16,8 +16,10 @@ single: crypt(3) pair: cipher; DES -.. deprecated:: 3.11 - The :mod:`crypt` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`crypt` module is deprecated + (see :pep:`PEP 594 <594#crypt>` for details and alternatives). + The :mod:`hashlib` module is a potential replacement for certain use cases. -------------- diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 6cca569a3f1c8..52950b551b756 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1360,10 +1360,6 @@ way is to instantiate one of the following classes: functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:type:`int` by default. - On Windows CE only the standard calling convention is used, for convenience the - :class:`WinDLL` and :class:`OleDLL` use the standard calling convention on this - platform. - The Python :term:`global interpreter lock` is released before calling any function exported by these libraries, and reacquired afterwards. @@ -1664,8 +1660,7 @@ See :ref:`ctypes-callback-functions` for examples. .. function:: WINFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) Windows only: The returned function prototype creates functions that use the - ``stdcall`` calling convention, except on Windows CE where - :func:`WINFUNCTYPE` is the same as :func:`CFUNCTYPE`. The function will + ``stdcall`` calling convention. The function will release the GIL during the call. *use_errno* and *use_last_error* have the same meaning as above. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index e0b28d7cb978d..bde24ec5f2e11 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -998,7 +998,7 @@ Other constructors, all class methods: ISO 8601 format, with the following exceptions: 1. Time zone offsets may have fractional seconds. - 2. The `T` separator may be replaced by any single unicode character. + 2. The ``T`` separator may be replaced by any single unicode character. 3. Ordinal dates are not currently supported. 4. Fractional hours and minutes are not supported. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 2ad84f20b5560..d052581c97012 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -571,9 +571,10 @@ Decimal objects >>> Decimal(321).exp() Decimal('2.561702493119680037517373933E+139') - .. method:: from_float(f) + .. classmethod:: from_float(f) - Classmethod that converts a float to a decimal number, exactly. + Alternative constructor that only accepts instances of :class:`float` or + :class:`int`. Note `Decimal.from_float(0.1)` is not the same as `Decimal('0.1')`. Since 0.1 is not exactly representable in binary floating point, the diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 08e6c736d3e3c..27a2f6bf79524 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -6,6 +6,12 @@ **Source code:** :source:`Lib/dis.py` +.. testsetup:: + + import dis + def myfunc(alist): + return len(alist) + -------------- The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by @@ -37,17 +43,18 @@ Example: Given the function :func:`myfunc`:: return len(alist) the following command can be used to display the disassembly of -:func:`myfunc`:: +:func:`myfunc`: - >>> dis.dis(myfunc) - 1 0 RESUME 0 +.. doctest:: - 2 2 PUSH_NULL - 4 LOAD_GLOBAL 1 (NULL + len) - 6 LOAD_FAST 0 (alist) - 8 PRECALL 1 - 10 CALL 1 - 12 RETURN_VALUE + >>> dis.dis(myfunc) + 2 0 RESUME 0 + + 3 2 LOAD_GLOBAL 1 (NULL + len) + 14 LOAD_FAST 0 (alist) + 16 PRECALL 1 + 20 CALL 1 + 30 RETURN_VALUE (The "2" is a line number). @@ -109,14 +116,15 @@ code. .. versionchanged:: 3.11 Added the ``show_caches`` parameter. -Example:: +Example: + +.. doctest:: >>> bytecode = dis.Bytecode(myfunc) >>> for instr in bytecode: ... print(instr.opname) ... RESUME - PUSH_NULL LOAD_GLOBAL LOAD_FAST PRECALL diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 0bbb640bea26b..2d50a49c6415e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -288,10 +288,6 @@ strings are treated as if they were docstrings. In output, a key ``K`` in Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. -.. impl-detail:: - Prior to version 3.4, extension modules written in C were not fully - searched by doctest. - .. _doctest-finding-examples: @@ -567,41 +563,35 @@ doctest decides whether actual output matches an example's expected output: .. data:: IGNORE_EXCEPTION_DETAIL - When specified, an example that expects an exception passes if an exception of - the expected type is raised, even if the exception detail does not match. For - example, an example expecting ``ValueError: 42`` will pass if the actual - exception raised is ``ValueError: 3*14``, but will fail, e.g., if - :exc:`TypeError` is raised. + When specified, doctests expecting exceptions pass so long as an exception + of the expected type is raised, even if the details + (message and fully-qualified exception name) don't match. + + For example, an example expecting ``ValueError: 42`` will pass if the actual + exception raised is ``ValueError: 3*14``, but will fail if, say, a + :exc:`TypeError` is raised instead. + It will also ignore any fully-qualified name included before the + exception class, which can vary between implementations and versions + of Python and the code/libraries in use. + Hence, all three of these variations will work with the flag specified: - It will also ignore the module name used in Python 3 doctest reports. Hence - both of these variations will work with the flag specified, regardless of - whether the test is run under Python 2.7 or Python 3.2 (or later versions):: + .. code-block:: pycon - >>> raise CustomError('message') + >>> raise Exception('message') Traceback (most recent call last): - CustomError: message + Exception: message - >>> raise CustomError('message') + >>> raise Exception('message') Traceback (most recent call last): - my_module.CustomError: message + builtins.Exception: message - Note that :const:`ELLIPSIS` can also be used to ignore the - details of the exception message, but such a test may still fail based - on whether or not the module details are printed as part of the - exception name. Using :const:`IGNORE_EXCEPTION_DETAIL` and the details - from Python 2.3 is also the only clear way to write a doctest that doesn't - care about the exception detail yet continues to pass under Python 2.3 or - earlier (those releases do not support :ref:`doctest directives - ` and ignore them as irrelevant comments). For example:: - - >>> (1, 2)[3] = 'moo' + >>> raise Exception('message') Traceback (most recent call last): - File "", line 1, in - TypeError: object doesn't support item assignment + __main__.Exception: message - passes under Python 2.3 and later Python versions with the flag specified, - even though the detail - changed in Python 2.4 to say "does not" instead of "doesn't". + Note that :const:`ELLIPSIS` can also be used to ignore the + details of the exception message, but such a test may still fail based + on whether the module name is present or matches exactly. .. versionchanged:: 3.2 :const:`IGNORE_EXCEPTION_DETAIL` now also ignores any information relating @@ -718,36 +708,51 @@ above. An example's doctest directives modify doctest's behavior for that single example. Use ``+`` to enable the named behavior, or ``-`` to disable it. -For example, this test passes:: +For example, this test passes: - >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE +.. doctest:: + :no-trim-doctest-flags: + + >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] Without the directive it would fail, both because the actual output doesn't have two blanks before the single-digit list elements, and because the actual output is on a single line. This test also passes, and also requires a directive to do -so:: +so: + +.. doctest:: + :no-trim-doctest-flags: - >>> print(list(range(20))) # doctest: +ELLIPSIS + >>> print(list(range(20))) # doctest: +ELLIPSIS [0, 1, ..., 18, 19] Multiple directives can be used on a single physical line, separated by -commas:: +commas: - >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE +.. doctest:: + :no-trim-doctest-flags: + + >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE [0, 1, ..., 18, 19] If multiple directive comments are used for a single example, then they are -combined:: +combined: + +.. doctest:: + :no-trim-doctest-flags: - >>> print(list(range(20))) # doctest: +ELLIPSIS - ... # doctest: +NORMALIZE_WHITESPACE + >>> print(list(range(20))) # doctest: +ELLIPSIS + ... # doctest: +NORMALIZE_WHITESPACE [0, 1, ..., 18, 19] As the previous example shows, you can add ``...`` lines to your example containing only directives. This can be useful when an example is too long for -a directive to comfortably fit on the same line:: +a directive to comfortably fit on the same line: + +.. doctest:: + :no-trim-doctest-flags: >>> print(list(range(5)) + list(range(10, 20)) + list(range(30, 40))) ... # doctest: +ELLIPSIS @@ -785,25 +790,25 @@ instead. Another is to do :: >>> d ['Harry', 'Hermione'] -.. note:: - - Before Python 3.6, when printing a dict, Python did not guarantee that - the key-value pairs was printed in any particular order. - There are others, but you get the idea. -Another bad idea is to print things that embed an object address, like :: +Another bad idea is to print things that embed an object address, like + +.. doctest:: - >>> id(1.0) # certain to fail some of the time + >>> id(1.0) # certain to fail some of the time # doctest: +SKIP 7948648 >>> class C: pass - >>> C() # the default repr() for instances embeds an address - <__main__.C instance at 0x00AC18F0> + >>> C() # the default repr() for instances embeds an address # doctest: +SKIP + + +The :const:`ELLIPSIS` directive gives a nice approach for the last example: -The :const:`ELLIPSIS` directive gives a nice approach for the last example:: +.. doctest:: + :no-trim-doctest-flags: - >>> C() #doctest: +ELLIPSIS - <__main__.C instance at 0x...> + >>> C() # doctest: +ELLIPSIS + Floating-point numbers are also subject to small output variations across platforms, because Python defers to the platform C library for float formatting, diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 3e1d97a03264b..98527cea43da2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -206,7 +206,7 @@ headers. The ``decoded`` value of the header will have all encoded words decoded to unicode. :class:`~encodings.idna` encoded domain names are also decoded to - unicode. The ``decoded`` value is set by :attr:`~str.join`\ ing the + unicode. The ``decoded`` value is set by :ref:`joining ` the :class:`str` value of the elements of the ``groups`` attribute with ``', '``. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 5829d4617893b..c3256c56c6366 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -126,11 +126,11 @@ Module Contents :func:`member` - Make `obj` a member. Can be used as a decorator. + Make ``obj`` a member. Can be used as a decorator. :func:`nonmember` - Do not make `obj` a member. Can be used as a decorator. + Do not make ``obj`` a member. Can be used as a decorator. .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index c893f2d5389d5..5f0ecf1f135ea 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -114,10 +114,10 @@ another rational number, or from a string. .. versionadded:: 3.8 - .. method:: from_float(flt) + .. classmethod:: from_float(flt) - This class method constructs a :class:`Fraction` representing the exact - value of *flt*, which must be a :class:`float`. Beware that + Alternative constructor which only accepts instances of + :class:`float` or :class:`numbers.Integral`. Beware that ``Fraction.from_float(0.3)`` is not the same value as ``Fraction(3, 10)``. .. note:: @@ -126,10 +126,10 @@ another rational number, or from a string. :class:`Fraction` instance directly from a :class:`float`. - .. method:: from_decimal(dec) + .. classmethod:: from_decimal(dec) - This class method constructs a :class:`Fraction` representing the exact - value of *dec*, which must be a :class:`decimal.Decimal` instance. + Alternative constructor which only accepts instances of + :class:`decimal.Decimal` or :class:`numbers.Integral`. .. note:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 83d54da72256a..f5e4c1a8640f4 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -500,6 +500,7 @@ are always available. They are listed here in alphabetical order. yield n, elem n += 1 +.. _func-eval: .. function:: eval(expression[, globals[, locals]]) @@ -859,8 +860,8 @@ are always available. They are listed here in alphabetical order. .. audit-event:: builtins.input/result result input - Raises an auditing event ``builtins.input/result`` with the result after - successfully reading input. + Raises an :ref:`auditing event ` ``builtins.input/result`` + with the result after successfully reading input. .. class:: int([x]) diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst index 084fef73daf28..318fe650776d2 100644 --- a/Doc/library/imghdr.rst +++ b/Doc/library/imghdr.rst @@ -7,8 +7,9 @@ **Source code:** :source:`Lib/imghdr.py` -.. deprecated:: 3.11 - The :mod:`imghdr` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`imghdr` module is deprecated + (see :pep:`PEP 594 <594#imghdr>` for details and alternatives). -------------- diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 121a730e0c9b4..000793a7e66ca 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -7,7 +7,7 @@ **Source code:** :source:`Lib/imp.py` -.. deprecated:: 3.4 +.. deprecated-removed:: 3.4 3.12 The :mod:`imp` module is deprecated in favor of :mod:`importlib`. .. index:: statement: import diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index b3740178d6f9c..d40ed70d5cd83 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -17,9 +17,9 @@ package metadata. Built in part on Python's import system, this library intends to replace similar functionality in the `entry point API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources` in Python 3.7 -and newer (backported as `importlib_resources`_ for older versions of -Python), this can eliminate the need to use the older and less efficient +:mod:`importlib.resources` (with new features backported to the +`importlib_resources`_ package), this can eliminate the need to use the older +and less efficient ``pkg_resources`` package. By "installed package" we generally mean a third-party package installed into diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 0241fb30b6efb..aac556e2c68d9 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -414,8 +414,8 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. versionchanged:: 3.5 - Starting in Python 3.6, this method will not be optional when + .. versionchanged:: 3.6 + This method is no longer optional when :meth:`exec_module` is defined. .. method:: exec_module(module) @@ -1250,6 +1250,9 @@ Checking if a module can be imported If you need to find out if a module can be imported without actually doing the import, then you should use :func:`importlib.util.find_spec`. + +Note that if ``name`` is a submodule (contains a dot), +:func:`importlib.util.find_spec` will import the parent module. :: import importlib.util @@ -1273,8 +1276,7 @@ import, then you should use :func:`importlib.util.find_spec`. Importing a source file directly '''''''''''''''''''''''''''''''' -To import a Python source file directly, use the following recipe -(Python 3.5 and newer only):: +To import a Python source file directly, use the following recipe:: import importlib.util import sys @@ -1355,9 +1357,7 @@ Import itself is implemented in Python code, making it possible to expose most of the import machinery through importlib. The following helps illustrate the various APIs that importlib exposes by providing an approximate implementation of -:func:`importlib.import_module` (Python 3.4 and newer for the importlib usage, -Python 3.6 and newer for other parts of the code). -:: +:func:`importlib.import_module`:: import importlib.util import sys diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 575b3088900e1..154d0f5dab0cd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -512,6 +512,7 @@ Retrieving source code If the documentation string for an object is not provided and the object is a class, a method, a property or a descriptor, retrieve the documentation string from the inheritance hierarchy. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. @@ -535,12 +536,14 @@ Retrieving source code .. function:: getmodule(object) - Try to guess which module an object was defined in. + Try to guess which module an object was defined in. Return ``None`` + if the module cannot be determined. .. function:: getsourcefile(object) - Return the name of the Python source file in which an object was defined. This + Return the name of the Python source file in which an object was defined + or ``None`` if no way can be identified to get the source. This will fail with a :exc:`TypeError` if the object is a built-in module, class, or function. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 26c9253cb7f5f..416c4eca5eb48 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -992,12 +992,7 @@ which incur interpreter overhead. "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) n = len(pool) - if r < 0 or r > n: - raise ValueError - c = 1 - k = min(r, n-r) - for i in range(1, k+1): - c = c * (n - k + i) // i + c = math.comb(n, r) if index < 0: index += c if index < 0 or index >= c: @@ -1071,6 +1066,7 @@ which incur interpreter overhead. >>> import operator >>> import collections + >>> import math >>> take(10, count()) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 510b0459bbb5a..1e203242327ca 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -125,13 +125,6 @@ See :ref:`json-commandline` for detailed documentation. This module's encoders and decoders preserve input and output order by default. Order is only lost if the underlying containers are unordered. - Prior to Python 3.7, :class:`dict` was not guaranteed to be ordered, so - inputs and outputs were typically scrambled unless - :class:`collections.OrderedDict` was specifically requested. Starting - with Python 3.7, the regular :class:`dict` became order preserving, so - it is no longer necessary to specify :class:`collections.OrderedDict` for - JSON generation and parsing. - Basic Usage ----------- diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 77a3e036841ba..112f0bae78daf 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -375,6 +375,8 @@ The :mod:`locale` module defines the following exception and functions: The default setting is determined by calling :func:`getdefaultlocale`. *category* defaults to :const:`LC_ALL`. + .. deprecated:: 3.11 3.13 + .. function:: strcoll(string1, string2) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 2f105ba29441d..310796e7ac6b1 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -716,7 +716,7 @@ root logger section is given below. The ``level`` entry can be one of ``DEBUG, INFO, WARNING, ERROR, CRITICAL`` or ``NOTSET``. For the root logger only, ``NOTSET`` means that all messages will be -logged. Level values are :func:`eval`\ uated in the context of the ``logging`` +logged. Level values are :ref:`evaluated ` in the context of the ``logging`` package's namespace. The ``handlers`` entry is a comma-separated list of handler names, which must @@ -763,13 +763,13 @@ handler. If blank, a default formatter (``logging._defaultFormatter``) is used. If a name is specified, it must appear in the ``[formatters]`` section and have a corresponding section in the configuration file. -The ``args`` entry, when :func:`eval`\ uated in the context of the ``logging`` +The ``args`` entry, when :ref:`evaluated ` in the context of the ``logging`` package's namespace, is the list of arguments to the constructor for the handler class. Refer to the constructors for the relevant handlers, or to the examples below, to see how typical entries are constructed. If not provided, it defaults to ``()``. -The optional ``kwargs`` entry, when :func:`eval`\ uated in the context of the +The optional ``kwargs`` entry, when :ref:`evaluated ` in the context of the ``logging`` package's namespace, is the keyword argument dict to the constructor for the handler class. If not provided, it defaults to ``{}``. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index ea6494f219ae7..3310c73656258 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -242,6 +242,10 @@ is the module's name in the Python package namespace. above example). In such circumstances, it is likely that specialized :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + If no handler is attached to this logger (or any of its ancestors, + taking into account the relevant :attr:`Logger.propagate` attributes), + the message will be sent to the handler set on :attr:`lastResort`. + .. versionchanged:: 3.2 The *stack_info* parameter was added. @@ -1038,6 +1042,10 @@ functions. above example). In such circumstances, it is likely that specialized :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + This function (as well as :func:`info`, :func:`warning`, :func:`error` and + :func:`critical`) will call :func:`basicConfig` if the root logger doesn't + have any handler attached. + .. versionchanged:: 3.2 The *stack_info* parameter was added. @@ -1080,16 +1088,6 @@ functions. Logs a message with level *level* on the root logger. The other arguments are interpreted as for :func:`debug`. - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. - .. function:: disable(level=CRITICAL) Provides an overriding level *level* for all loggers which takes precedence over diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst index 416b181f45a77..bfaedb4609199 100644 --- a/Doc/library/mailcap.rst +++ b/Doc/library/mailcap.rst @@ -7,9 +7,10 @@ **Source code:** :source:`Lib/mailcap.py` -.. deprecated:: 3.11 - The :mod:`mailcap` module is deprecated. See :pep:`594` for the rationale - and the :mod:`mimetypes` module for an alternative. +.. deprecated-removed:: 3.11 3.13 + The :mod:`mailcap` module is deprecated + (see :pep:`PEP 594 <594#mailcap>` for details). + The :mod:`mimetypes` module provides an alternative. -------------- @@ -59,6 +60,18 @@ standard. However, mailcap files are supported on most Unix systems. use) to determine whether or not the mailcap line applies. :func:`findmatch` will automatically check such conditions and skip the entry if the check fails. + .. versionchanged:: 3.11 + + To prevent security issues with shell metacharacters (symbols that have + special effects in a shell command line), ``findmatch`` will refuse + to inject ASCII characters other than alphanumerics and ``@+=:,./-_`` + into the returned command line. + + If a disallowed character appears in *filename*, ``findmatch`` will always + return ``(None, None)`` as if no entry was found. + If such a character appears elsewhere (a value in *plist* or in *MIMEtype*), + ``findmatch`` will ignore all mailcap entries which use that value. + A :mod:`warning ` will be raised in either case. .. function:: getcaps() diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst index b2fa20873fbbf..fbe55db937230 100644 --- a/Doc/library/msilib.rst +++ b/Doc/library/msilib.rst @@ -13,8 +13,9 @@ .. index:: single: msi -.. deprecated:: 3.11 - The :mod:`msilib` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`msilib` module is deprecated + (see :pep:`PEP 594 <594#msilib>` for details). -------------- diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 70802ee1fdecb..2a66b0f65c088 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1666,6 +1666,7 @@ different machines. A manager object controls a server process which manages proxies. .. function:: multiprocessing.Manager() + :module: Returns a started :class:`~multiprocessing.managers.SyncManager` object which can be used for sharing objects between processes. The returned manager diff --git a/Doc/library/nis.rst b/Doc/library/nis.rst index f6b6ea83946b0..fd3c3d9293d24 100644 --- a/Doc/library/nis.rst +++ b/Doc/library/nis.rst @@ -10,8 +10,9 @@ .. moduleauthor:: Fred Gansevles .. sectionauthor:: Moshe Zadka -.. deprecated:: 3.11 - The :mod:`nis` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`nis` module is deprecated + (see :pep:`PEP 594 <594#nis>` for details). -------------- diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index b77845ed0dee9..b12f82ed75a6f 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -202,9 +202,9 @@ forward and reverse instances of any given operator. For example, if isinstance(a, Rational): # Includes ints. return monomorphic_operator(a, b) - elif isinstance(a, numbers.Real): + elif isinstance(a, Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): + elif isinstance(a, Complex): return fallback_operator(complex(a), complex(b)) else: return NotImplemented diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index b1094198f4c84..0d686b10365a6 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -131,7 +131,7 @@ option These option syntaxes are not supported by :mod:`optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting - VMS, MS-DOS, and/or Windows. + Windows or certain legacy platforms (e.g. VMS, MS-DOS). option argument an argument that follows an option, is closely associated with that option, diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index c201b1460ede3..ce7913e3712d7 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -5,7 +5,7 @@ :synopsis: Operations on pathnames. **Source code:** :source:`Lib/posixpath.py` (for POSIX) and -:source:`Lib/ntpath.py` (for Windows NT). +:source:`Lib/ntpath.py` (for Windows). .. index:: single: path; operations diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 3c189bb40e234..dc0f2e4158ac0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3916,13 +3916,13 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix. -.. function:: popen(cmd, mode='r', buffering=-1, encoding=None) +.. function:: popen(cmd, mode='r', buffering=-1) Open a pipe to or from command *cmd*. The return value is an open file object connected to the pipe, which can be read or written depending on whether *mode* is ``'r'`` (default) or ``'w'``. - The *buffering* and *encoding* arguments have the same meaning as + The *buffering* argument have the same meaning as the corresponding argument to the built-in :func:`open` function. The returned file object reads or writes text strings rather than bytes. @@ -3945,8 +3945,13 @@ written in Python, such as a mail server's external command delivery program. documentation for more powerful ways to manage and communicate with subprocesses. - .. versionchanged:: 3.11 - Added the *encoding* parameter. + .. note:: + The :ref:`Python UTF-8 Mode ` affects encodings used + for *cmd* and pipe contents. + + :func:`popen` is a simple wrapper around :class:`subprocess.Popen`. + Use :class:`subprocess.Popen` or :func:`subprocess.run` to + control options like encodings. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst index e0f0a6b8259e4..e14c1bf8d5367 100644 --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -6,8 +6,9 @@ :synopsis: Access to OSS-compatible audio devices. :deprecated: -.. deprecated:: 3.11 - The :mod:`ossaudiodev` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`ossaudiodev` module is deprecated + (see :pep:`PEP 594 <594#ossaudiodev>` for details). -------------- diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index ab26e2f1719fe..7c407242515d2 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1021,8 +1021,9 @@ call fails (for example because the path doesn't exist). Rename this file or directory to the given *target*, and return a new Path instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. *target* can be - either a string or another path object:: + it will be replaced silently if the user has permission. + On Windows, if *target* exists, :exc:`FileExistsError` will be raised. + *target* can be either a string or another path object:: >>> p = Path('foo') >>> p.open('w').write('some text') diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ca59576336bf8..383c3adcf289d 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -233,7 +233,8 @@ Multiple commands may be entered on a single line, separated by ``;;``. (A single ``;`` is not used as it is the separator for multiple commands in a line that is passed to the Python parser.) No intelligence is applied to separating the commands; the input is split at the first ``;;`` pair, even if it is in the -middle of a quoted string. +middle of a quoted string. A workaround for strings with double semicolons +is to use implicit string concatenation ``';'';'`` or ``";"";"``. .. index:: pair: .pdbrc; file diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index fa14b64027e8f..41b0f48f4611d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -502,10 +502,10 @@ The following types can be pickled: * tuples, lists, sets, and dictionaries containing only picklable objects; -* functions (built-in and user-defined) defined at the top level of a module - (using :keyword:`def`, not :keyword:`lambda`); +* functions (built-in and user-defined) accessible from the top level of a + module (using :keyword:`def`, not :keyword:`lambda`); -* classes defined at the top level of a module; +* classes accessible from the top level of a module; * instances of such classes whose the result of calling :meth:`__getstate__` is picklable (see section :ref:`pickle-inst` for details). @@ -517,9 +517,9 @@ structure may exceed the maximum recursion depth, a :exc:`RecursionError` will b raised in this case. You can carefully raise this limit with :func:`sys.setrecursionlimit`. -Note that functions (built-in and user-defined) are pickled by fully qualified -name, not by value. [#]_ This means that only the function name is -pickled, along with the name of the module the function is defined in. Neither +Note that functions (built-in and user-defined) are pickled by fully +:term:`qualified name`, not by value. [#]_ This means that only the function name is +pickled, along with the name of the containing module and classes. Neither the function's code, nor any of its function attributes are pickled. Thus the defining module must be importable in the unpickling environment, and the module must contain the named object, otherwise an exception will be raised. [#]_ diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst index 4de8c51bcae04..245dd0d252088 100644 --- a/Doc/library/pipes.rst +++ b/Doc/library/pipes.rst @@ -10,8 +10,10 @@ **Source code:** :source:`Lib/pipes.py` -.. deprecated:: 3.11 - The :mod:`pipes` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`pipes` module is deprecated + (see :pep:`PEP 594 <594#pipes>` for details). + Please use the :mod:`subprocess` module instead. -------------- diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 3cd9f252fee6f..2392785d6c465 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -7,7 +7,7 @@ .. moduleauthor:: Fredrik Lundh .. sectionauthor:: Andrew M. Kuchling -**Source code:** :source:`Lib/re.py` +**Source code:** :source:`Lib/re/` -------------- @@ -665,40 +665,14 @@ functions are simplified versions of the full featured methods for compiled regular expressions. Most non-trivial applications always use the compiled form. + +Flags +^^^^^ + .. versionchanged:: 3.6 Flag constants are now instances of :class:`RegexFlag`, which is a subclass of :class:`enum.IntFlag`. -.. function:: compile(pattern, flags=0) - - Compile a regular expression pattern into a :ref:`regular expression object - `, which can be used for matching using its - :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described - below. - - The expression's behaviour can be modified by specifying a *flags* value. - Values can be any of the following variables, combined using bitwise OR (the - ``|`` operator). - - The sequence :: - - prog = re.compile(pattern) - result = prog.match(string) - - is equivalent to :: - - result = re.match(pattern, string) - - but using :func:`re.compile` and saving the resulting regular expression - object for reuse is more efficient when the expression will be used several - times in a single program. - - .. note:: - - The compiled versions of the most recent patterns passed to - :func:`re.compile` and the module-level matching functions are cached, so - programs that use only a few regular expressions at a time needn't worry - about compiling regular expressions. .. class:: RegexFlag @@ -823,6 +797,41 @@ form. Corresponds to the inline flag ``(?x)``. +Functions +^^^^^^^^^ + +.. function:: compile(pattern, flags=0) + + Compile a regular expression pattern into a :ref:`regular expression object + `, which can be used for matching using its + :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described + below. + + The expression's behaviour can be modified by specifying a *flags* value. + Values can be any of the following variables, combined using bitwise OR (the + ``|`` operator). + + The sequence :: + + prog = re.compile(pattern) + result = prog.match(string) + + is equivalent to :: + + result = re.match(pattern, string) + + but using :func:`re.compile` and saving the resulting regular expression + object for reuse is more efficient when the expression will be used several + times in a single program. + + .. note:: + + The compiled versions of the most recent patterns passed to + :func:`re.compile` and the module-level matching functions are cached, so + programs that use only a few regular expressions at a time needn't worry + about compiling regular expressions. + + .. function:: search(pattern, string, flags=0) Scan through *string* looking for the first location where the regular expression @@ -1058,6 +1067,9 @@ form. Clear the regular expression cache. +Exceptions +^^^^^^^^^^ + .. exception:: error(msg, pattern=None, pos=None) Exception raised when a string passed to one of the functions here is not a @@ -1312,6 +1324,14 @@ Match objects support the following methods and attributes: >>> m[2] # The second parenthesized subgroup. 'Newton' + Named groups are supported as well:: + + >>> m = re.match(r"(?P\w+) (?P\w+)", "Isaac Newton") + >>> m['first_name'] + 'Isaac' + >>> m['last_name'] + 'Newton' + .. versionadded:: 3.6 diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index 6b37a0517063d..a0d1fb0aa5152 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -14,8 +14,9 @@ This module offers several classes to implement SMTP (email) servers. -.. deprecated:: 3.6 - :mod:`smtpd` will be removed in Python 3.12 (:pep:`594`). +.. deprecated-removed:: 3.6 3.12 + The :mod:`smtpd` module is deprecated + (see :pep:`PEP 594 <594#smtpd>` for details). The `aiosmtpd `_ package is a recommended replacement for this module. It is based on :mod:`asyncio` and provides a more straightforward API. diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst index 41bce18b9cd84..e1dbe4a1a3448 100644 --- a/Doc/library/sndhdr.rst +++ b/Doc/library/sndhdr.rst @@ -14,8 +14,9 @@ single: A-LAW single: u-LAW -.. deprecated:: 3.11 - The :mod:`sndhdr` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`sndhdr` module is deprecated + (see :pep:`PEP 594 <594#sndhdr>` for details and alternatives). -------------- diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst old mode 100755 new mode 100644 index ee1aee5278b3b..c27f408890ac3 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -233,9 +233,9 @@ resolution and/or the host configuration. For deterministic behavior use a numeric address in *host* portion. All errors raise exceptions. The normal exceptions for invalid argument types -and out-of-memory conditions can be raised; starting from Python 3.3, errors +and out-of-memory conditions can be raised. Errors related to socket or address semantics raise :exc:`OSError` or one of its -subclasses (they used to raise :exc:`socket.error`). +subclasses. Non-blocking mode is supported through :meth:`~socket.setblocking`. A generalization of this based on timeouts is supported through diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst index cb31a10a52e00..87e09167ada42 100644 --- a/Doc/library/spwd.rst +++ b/Doc/library/spwd.rst @@ -6,8 +6,9 @@ :synopsis: The shadow password database (getspnam() and friends). :deprecated: -.. deprecated:: 3.11 - The :mod:`spwd` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`spwd` module is deprecated + (see :pep:`PEP 594 <594#spwd>` for details and alternatives). -------------- diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 69e77e922a9ab..1843e22640f01 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -518,22 +518,19 @@ Connection Objects .. method:: create_collation(name, callable) - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. + Create a collation named *name* using the collating function *callable*. + *callable* is passed two :class:`string ` arguments, + and it should return an :class:`integer `: - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. + * ``1`` if the first is ordered higher than the second + * ``-1`` if the first is ordered lower than the second + * ``0`` if they are ordered equal - The following example shows a custom collation that sorts "the wrong way": + The following example shows a reverse sorting collation: .. literalinclude:: ../includes/sqlite3/collation_reverse.py - To remove a collation, call ``create_collation`` with ``None`` as callable:: - - con.create_collation("reverse", None) + Remove a collation function by setting *callable* to :const:`None`. .. versionchanged:: 3.11 The collation name can contain any Unicode character. Earlier, only diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 54bd335282699..c7ca4395ed96c 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -714,7 +714,7 @@ Constants Selects SSL version 2 as the channel encryption protocol. This protocol is not available if OpenSSL is compiled with the - ``OPENSSL_NO_SSL2`` flag. + ``no-ssl2`` option. .. warning:: @@ -728,8 +728,8 @@ Constants Selects SSL version 3 as the channel encryption protocol. - This protocol is not be available if OpenSSL is compiled with the - ``OPENSSL_NO_SSLv3`` flag. + This protocol is not available if OpenSSL is compiled with the + ``no-ssl3`` option. .. warning:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 44447400c29bc..33fd2831228f8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1866,6 +1866,8 @@ expression support in the :mod:`re` module). +.. _meth-str-join: + .. method:: str.join(iterable) Return a string which is the concatenation of the strings in *iterable*. @@ -2573,16 +2575,6 @@ The representation of bytes objects uses the literal format (``b'...'``) since it is often more useful than e.g. ``bytes([46, 46, 46])``. You can always convert a bytes object into a list of integers using ``list(b)``. -.. note:: - For Python 2.x users: In the Python 2.x series, a variety of implicit - conversions between 8-bit strings (the closest thing 2.x offers to a - built-in binary data type) and Unicode strings were permitted. This was a - backwards compatibility workaround to account for the fact that Python - originally only supported 8-bit text, and Unicode text was a later - addition. In Python 3.x, those implicit conversions are gone - conversions - between 8-bit binary data and Unicode text must be explicit, and bytes and - string objects will always compare unequal. - .. _typebytearray: diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ce581abdd1dce..4031a5f62167f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -33,9 +33,6 @@ The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the underlying :class:`Popen` interface can be used directly. -The :func:`run` function was added in Python 3.5; if you need to retain -compatibility with older versions, see the :ref:`call-function-trio` section. - .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ capture_output=False, shell=False, cwd=None, timeout=None, \ diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index cfb1257f58548..c7a38d96ade13 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -9,8 +9,9 @@ **Source code:** :source:`Lib/sunau.py` -.. deprecated:: 3.11 - The :mod:`sunau` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`sunau` module is deprecated + (see :pep:`PEP 594 <594#sunau>` for details). -------------- diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 97b0a713e4422..70b8c7d1511d0 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -11,8 +11,9 @@ .. index:: single: protocol; Telnet -.. deprecated:: 3.11 - The :mod:`telnetlib` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`telnetlib` module is deprecated + (see :pep:`PEP 594 <594#telnetlib>` for details and alternatives). -------------- diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 8fe38b5e81c08..b7e604c1b70ac 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -84,7 +84,7 @@ The module defines the following user-callable items: file-like object. Whether the name can be used to open the file a second time, while the named temporary file is still open, varies across platforms (it can be so used on Unix; it cannot - on Windows NT or later). If *delete* is true (the default), the file is + on Windows). If *delete* is true (the default), the file is deleted as soon as it is closed. The returned object is always a file-like object whose :attr:`!file` attribute is the underlying true file object. This file-like object can diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 3b0cb60f87452..fb1ff567d49e5 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -85,11 +85,11 @@ The module defines the following functions: .. function:: tcsetwinsize(fd, winsize) - Set the tty window size for file descriptor *fd* from *winsize*, which is - a two-item tuple ``(ws_row, ws_col)`` like the one returned by - :func:`tcgetwinsize`. Requires at least one of the pairs - (:const:`termios.TIOCGWINSZ`, :const:`termios.TIOCSWINSZ`); - (:const:`termios.TIOCGSIZE`, :const:`termios.TIOCSSIZE`) to be defined. + Set the tty window size for file descriptor *fd* from *winsize*, which is + a two-item tuple ``(ws_row, ws_col)`` like the one returned by + :func:`tcgetwinsize`. Requires at least one of the pairs + (:const:`termios.TIOCGWINSZ`, :const:`termios.TIOCSWINSZ`); + (:const:`termios.TIOCGSIZE`, :const:`termios.TIOCSSIZE`) to be defined. .. versionadded:: 3.11 diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 699db14596f25..72411007e2452 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -359,13 +359,19 @@ The :mod:`test.support` module defines the following constants: .. data:: MISSING_C_DOCSTRINGS - Return ``True`` if running on CPython, not on Windows, and configuration - not set with ``WITH_DOC_STRINGS``. + Set to ``True`` if Python is built without docstrings (the + :c:macro:`WITH_DOC_STRINGS` macro is not defined). + See the :option:`configure --without-doc-strings <--without-doc-strings>` option. + + See also the :data:`HAVE_DOCSTRINGS` variable. .. data:: HAVE_DOCSTRINGS - Check for presence of docstrings. + Set to ``True`` if function docstrings are available. + See the :option:`python -OO <-O>` option, which strips docstrings of functions implemented in Python. + + See also the :data:`MISSING_C_DOCSTRINGS` variable. .. data:: TEST_HTTP_URL @@ -423,11 +429,6 @@ The :mod:`test.support` module defines the following functions: Used when tests are executed by :mod:`test.regrtest`. -.. function:: system_must_validate_cert(f) - - Raise :exc:`unittest.SkipTest` on TLS certification validation failures. - - .. function:: sortdict(dict) Return a repr of *dict* with keys sorted. @@ -445,12 +446,12 @@ The :mod:`test.support` module defines the following functions: .. function:: match_test(test) - Match *test* to patterns set in :func:`set_match_tests`. + Determine whether *test* matches the patterns set in :func:`set_match_tests`. -.. function:: set_match_tests(patterns) +.. function:: set_match_tests(accept_patterns=None, ignore_patterns=None) - Define match test with regular expression *patterns*. + Define match patterns on test filenames and test method names for filtering tests. .. function:: run_unittest(*classes) @@ -490,7 +491,9 @@ The :mod:`test.support` module defines the following functions: .. function:: check_impl_detail(**guards) Use this check to guard CPython's implementation-specific tests or to - run them only on the implementations guarded by the arguments:: + run them only on the implementations guarded by the arguments. This + function returns ``True`` or ``False`` depending on the host platform. + Example usage:: check_impl_detail() # Only on CPython (default). check_impl_detail(jython=True) # Only on Jython. @@ -509,7 +512,7 @@ The :mod:`test.support` module defines the following functions: time the regrtest began. -.. function:: get_original_stdout +.. function:: get_original_stdout() Return the original stdout set by :func:`record_original_stdout` or ``sys.stdout`` if it's not set. @@ -554,7 +557,7 @@ The :mod:`test.support` module defines the following functions: .. function:: disable_faulthandler() - A context manager that replaces ``sys.stderr`` with ``sys.__stderr__``. + A context manager that temporary disables :mod:`faulthandler`. .. function:: gc_collect() @@ -567,8 +570,8 @@ The :mod:`test.support` module defines the following functions: .. function:: disable_gc() - A context manager that disables the garbage collector upon entry and - reenables it upon exit. + A context manager that disables the garbage collector on entry. On + exit, the garbage collector is restored to its prior state. .. function:: swap_attr(obj, attr, new_val) @@ -642,14 +645,14 @@ The :mod:`test.support` module defines the following functions: .. function:: calcobjsize(fmt) - Return :func:`struct.calcsize` for ``nP{fmt}0n`` or, if ``gettotalrefcount`` - exists, ``2PnP{fmt}0P``. + Return the size of the :c:type:`PyObject` whose structure members are + defined by *fmt*. The returned value includes the size of the Python object header and alignment. .. function:: calcvobjsize(fmt) - Return :func:`struct.calcsize` for ``nPn{fmt}0n`` or, if ``gettotalrefcount`` - exists, ``2PnPn{fmt}0P``. + Return the size of the :c:type:`PyVarObject` whose structure members are + defined by *fmt*. The returned value includes the size of the Python object header and alignment. .. function:: checksizeof(test, o, size) @@ -665,6 +668,11 @@ The :mod:`test.support` module defines the following functions: have an associated comment identifying the relevant tracker issue. +.. function:: system_must_validate_cert(f) + + A decorator that skips the decorated test on TLS certification validation failures. + + .. decorator:: run_with_locale(catstr, *locales) A decorator for running a function in a different locale, correctly @@ -682,19 +690,19 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_freebsd_version(*min_version) Decorator for the minimum version when running test on FreeBSD. If the - FreeBSD version is less than the minimum, raise :exc:`unittest.SkipTest`. + FreeBSD version is less than the minimum, the test is skipped. .. decorator:: requires_linux_version(*min_version) Decorator for the minimum version when running test on Linux. If the - Linux version is less than the minimum, raise :exc:`unittest.SkipTest`. + Linux version is less than the minimum, the test is skipped. .. decorator:: requires_mac_version(*min_version) Decorator for the minimum version when running test on macOS. If the - macOS version is less than the minimum, raise :exc:`unittest.SkipTest`. + macOS version is less than the minimum, the test is skipped. .. decorator:: requires_IEEE_754 @@ -732,7 +740,7 @@ The :mod:`test.support` module defines the following functions: Decorator for only running the test if :data:`HAVE_DOCSTRINGS`. -.. decorator:: cpython_only(test) +.. decorator:: cpython_only Decorator for tests only applicable to CPython. @@ -743,12 +751,12 @@ The :mod:`test.support` module defines the following functions: returns ``False``, then uses *msg* as the reason for skipping the test. -.. decorator:: no_tracing(func) +.. decorator:: no_tracing Decorator to temporarily turn off tracing for the duration of the test. -.. decorator:: refcount_test(test) +.. decorator:: refcount_test Decorator for tests which involve reference counting. The decorator does not run the test if it is not run by CPython. Any trace function is unset @@ -771,10 +779,9 @@ The :mod:`test.support` module defines the following functions: means the test doesn't support dummy runs when ``-M`` is not specified. -.. decorator:: bigaddrspacetest(f) +.. decorator:: bigaddrspacetest - Decorator for tests that fill the address space. *f* is the function to - wrap. + Decorator for tests that fill the address space. .. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) @@ -876,7 +883,7 @@ The :mod:`test.support` module defines the following functions: .. function:: check_free_after_iterating(test, iter, cls, args=()) - Assert that *iter* is deallocated after iterating. + Assert instances of *cls* are deallocated after iterating. .. function:: missing_compiler_executable(cmd_names=[]) @@ -967,6 +974,16 @@ The :mod:`test.support` module defines the following classes: Class to save and restore signal handlers registered by the Python signal handler. + .. method:: save(self) + + Save the signal handlers to a dictionary mapping signal numbers to the + current signal handler. + + .. method:: restore(self) + + Set the signal numbers from the :meth:`save` dictionary to the saved + handler. + .. class:: Matcher() @@ -1110,11 +1127,11 @@ script execution tests. variables *env_vars* succeeds (``rc == 0``) and return a ``(return code, stdout, stderr)`` tuple. - If the ``__cleanenv`` keyword is set, *env_vars* is used as a fresh + If the *__cleanenv* keyword-only parameter is set, *env_vars* is used as a fresh environment. Python is started in isolated mode (command line option ``-I``), - except if the ``__isolated`` keyword is set to ``False``. + except if the *__isolated* keyword-only parameter is set to ``False``. .. versionchanged:: 3.9 The function no longer strips whitespaces from *stderr*. @@ -1225,15 +1242,17 @@ The :mod:`test.support.threading_helper` module provides support for threading t is still alive after *timeout* seconds. -.. decorator:: reap_threads(func) +.. decorator:: reap_threads Decorator to ensure the threads are cleaned up even if the test fails. .. function:: start_threads(threads, unlock=None) - Context manager to start *threads*. It attempts to join the threads upon - exit. + Context manager to start *threads*, which is a sequence of threads. + *unlock* is a function called after the threads are started, even if an + exception was raised; an example would be :meth:`threading.Event.set`. + ``start_threads`` will attempt to join the started threads upon exit. .. function:: threading_cleanup(*original_values) @@ -1315,7 +1334,10 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. data:: TESTFN_NONASCII - Set to a filename containing the :data:`FS_NONASCII` character. + Set to a filename containing the :data:`FS_NONASCII` character, if it exists. + This guarantees that if the filename exists, it can be encoded and decoded + with the default filesystem encoding. This allows tests that require a + non-ASCII filename to be easily skipped on platforms where they can't work. .. data:: TESTFN_UNENCODABLE @@ -1413,13 +1435,16 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: rmdir(filename) Call :func:`os.rmdir` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence of the file. + wrapped with a wait loop that checks for the existence of the file, + which is needed due to antivirus programs that can hold files open and prevent + deletion. .. function:: rmtree(path) Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and - :func:`os.rmdir` to remove a path and its contents. On Windows platforms, + :func:`os.rmdir` to remove a path and its contents. As with :func:`rmdir`, + on Windows platforms this is wrapped with a wait loop that checks for the existence of the files. @@ -1466,7 +1491,8 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: unlink(filename) - Call :func:`os.unlink` on *filename*. On Windows platforms, this is + Call :func:`os.unlink` on *filename*. As with :func:`rmdir`, + on Windows platforms, this is wrapped with a wait loop that checks for the existence of the file. @@ -1523,7 +1549,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. .. versionadded:: 3.1 -.. function:: import_module(name, deprecated=False, *, required_on()) +.. function:: import_module(name, deprecated=False, *, required_on=()) This function imports and returns the named module. Unlike a normal import, this function raises :exc:`unittest.SkipTest` if the module @@ -1565,7 +1591,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. A context manager to force import to return a new module reference. This is useful for testing module-level behaviors, such as the emission of a - DeprecationWarning on import. Example usage:: + :exc:`DeprecationWarning` on import. Example usage:: with CleanImport('foo'): importlib.import_module('foo') # New reference. @@ -1573,7 +1599,7 @@ The :mod:`test.support.import_helper` module provides support for import tests. .. class:: DirsOnSysPath(*paths) - A context manager to temporarily add directories to sys.path. + A context manager to temporarily add directories to :data:`sys.path`. This makes a copy of :data:`sys.path`, appends any directories given as positional arguments, then reverts :data:`sys.path` to the copied diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b777560961690..e654dedfd91a2 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -293,7 +293,7 @@ There is the possibility that "dummy thread objects" are created. These are thread objects corresponding to "alien threads", which are threads of control started outside the threading module, such as directly from C code. Dummy thread objects have limited functionality; they are always considered alive and -daemonic, and cannot be :meth:`~Thread.join`\ ed. They are never deleted, +daemonic, and cannot be :ref:`joined `. They are never deleted, since it is impossible to detect the termination of alien threads. @@ -366,6 +366,8 @@ since it is impossible to detect the termination of alien threads. >>> t.run() 1 + .. _meth-thread-join: + .. method:: join(timeout=None) Wait until the thread terminates. This blocks the calling thread until @@ -383,7 +385,7 @@ since it is impossible to detect the termination of alien threads. When the *timeout* argument is not present or ``None``, the operation will block until the thread terminates. - A thread can be :meth:`~Thread.join`\ ed many times. + A thread can be joined many times. :meth:`~Thread.join` raises a :exc:`RuntimeError` if an attempt is made to join the current thread as that would cause a deadlock. It is also diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 65395d824cab7..096a343bd9558 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -124,16 +124,72 @@ the modern themed widget set and API:: from tkinter import ttk -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=1) - - The :class:`Tk` class is instantiated without arguments. This creates a toplevel - widget of Tk which usually is the main window of an application. Each instance - has its own associated Tcl interpreter. - - .. FIXME: The following keyword arguments are currently recognized: - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=0) +.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) + + Construct a toplevel Tk widget, which is usually the main window of an + application, and initialize a Tcl interpreter for this widget. Each + instance has its own associated Tcl interpreter. + + The :class:`Tk` class is typically instantiated using all default values. + However, the following keyword arguments are currently recognized: + + *screenName* + When given (as a string), sets the :envvar:`DISPLAY` environment + variable. (X11 only) + *baseName* + Name of the profile file. By default, *baseName* is derived from the + program name (``sys.argv[0]``). + *className* + Name of the widget class. Used as a profile file and also as the name + with which Tcl is invoked (*argv0* in *interp*). + *useTk* + If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` + function sets this to ``False``. + *sync* + If ``True``, execute all X server commands synchronously, so that errors + are reported immediately. Can be used for debugging. (X11 only) + *use* + Specifies the *id* of the window in which to embed the application, + instead of it being created as an independent toplevel window. *id* must + be specified in the same way as the value for the -use option for + toplevel widgets (that is, it has a form like that returned by + :meth:`winfo_id`). + + Note that on some platforms this will only work correctly if *id* refers + to a Tk frame or toplevel that has its -container option enabled. + + :class:`Tk` reads and interprets profile files, named + :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl + interpreter and calls :func:`exec` on the contents of + :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the + profile files is the :envvar:`HOME` environment variable or, if that + isn't defined, then :attr:`os.curdir`. + + .. attribute:: tk + + The Tk application object created by instantiating :class:`Tk`. This + provides access to the Tcl interpreter. Each widget that is attached + the same instance of :class:`Tk` has the same value for its :attr:`tk` + attribute. + + .. attribute:: master + + The widget object that contains this widget. For :class:`Tk`, the + *master* is :const:`None` because it is the main window. The terms + *master* and *parent* are similar and sometimes used interchangeably + as argument names; however, calling :meth:`winfo_parent` returns a + string of the widget name whereas :attr:`master` returns the object. + *parent*/*child* reflects the tree-like relationship while + *master*/*slave* reflects the container structure. + + .. attribute:: children + + The immediate descendants of this widget as a :class:`dict` with the + child widget names as the keys and the child instance objects as the + values. + + +.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) The :func:`Tcl` function is a factory function which creates an object much like that created by the :class:`Tk` class, except that it does not initialize the Tk @@ -821,8 +877,9 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the bind man page and - page 201 of John Ousterhout's book for details). + is a string that denotes the target kind of event. (See the + :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, + :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). func is a Python function, taking one argument, to be invoked when the event occurs. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 005a515234110..c7c5536a64fb4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -78,6 +78,8 @@ annotations. These include: *Introducing* :data:`TypeVarTuple` * :pep:`647`: User-Defined Type Guards *Introducing* :data:`TypeGuard` +* :pep:`655`: Marking individual TypedDict items as required or potentially-missing + *Introducing* :data:`Required` and :data:`NotRequired` * :pep:`673`: Self type *Introducing* :data:`Self` * :pep:`675`: Arbitrary Literal String Type @@ -1022,6 +1024,18 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.8 +.. data:: Required + +.. data:: NotRequired + + Special typing constructs that mark individual keys of a :class:`TypedDict` + as either required or non-required respectively. + + For more information, see :class:`TypedDict` and + :pep:`655` ("Marking individual TypedDict items as required or potentially-missing"). + + .. versionadded:: 3.11 + .. data:: Annotated A type, introduced in :pep:`593` (``Flexible function and variable @@ -1706,8 +1720,21 @@ These are not used in annotations. They are building blocks for declaring types. Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) By default, all keys must be present in a ``TypedDict``. It is possible to - override this by specifying totality. - Usage:: + mark individual keys as non-required using :data:`NotRequired`:: + + class Point2D(TypedDict): + x: int + y: int + label: NotRequired[str] + + # Alternative syntax + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) + + This means that a ``Point2D`` ``TypedDict`` can have the ``label`` + key omitted. + + It is also possible to mark all keys as non-required by default + by specifying a totality of ``False``:: class Point2D(TypedDict, total=False): x: int @@ -1721,6 +1748,21 @@ These are not used in annotations. They are building blocks for declaring types. ``True`` as the value of the ``total`` argument. ``True`` is the default, and makes all items defined in the class body required. + Individual keys of a ``total=False`` ``TypedDict`` can be marked as + required using :data:`Required`:: + + class Point2D(TypedDict, total=False): + x: Required[int] + y: Required[int] + label: str + + # Alternative syntax + Point2D = TypedDict('Point2D', { + 'x': Required[int], + 'y': Required[int], + 'label': str + }, total=False) + It is possible for a ``TypedDict`` type to inherit from one or more other ``TypedDict`` types using the class-based syntax. Usage:: @@ -1785,11 +1827,16 @@ These are not used in annotations. They are building blocks for declaring types. ``Point2D.__required_keys__`` and ``Point2D.__optional_keys__`` return :class:`frozenset` objects containing required and non-required keys, respectively. - Currently the only way to declare both required and non-required keys in the - same ``TypedDict`` is mixed inheritance, declaring a ``TypedDict`` with one value - for the ``total`` argument and then inheriting it from another ``TypedDict`` with - a different value for ``total``. - Usage:: + + Keys marked with :data:`Required` will always appear in ``__required_keys__`` + and keys marked with :data:`NotRequired` will always appear in ``__optional_keys__``. + + For backwards compatibility with Python 3.10 and below, + it is also possible to use inheritance to declare both required and + non-required keys in the same ``TypedDict`` . This is done by declaring a + ``TypedDict`` with one value for the ``total`` argument and then + inheriting from it in another ``TypedDict`` with a different value for + ``total``:: >>> class Point2D(TypedDict, total=False): ... x: int @@ -1807,6 +1854,10 @@ These are not used in annotations. They are building blocks for declaring types. .. versionadded:: 3.8 + .. versionchanged:: 3.11 + Added support for marking individual keys as :data:`Required` or :data:`NotRequired`. + See :pep:`655`. + .. versionchanged:: 3.11 Added support for generic ``TypedDict``\ s. @@ -1978,7 +2029,8 @@ Other concrete types .. deprecated:: 3.11 Python 2 is no longer supported, and most type checkers also no longer - support type checking Python 2 code. Users should now use + support type checking Python 2 code. Removal of the alias is not + currently planned, but users are encouraged to use :class:`str` instead of ``Text`` wherever possible. Abstract Base Classes @@ -2428,6 +2480,75 @@ Functions and decorators .. versionadded:: 3.11 +.. decorator:: dataclass_transform + + :data:`~typing.dataclass_transform` may be used to + decorate a class, metaclass, or a function that is itself a decorator. + The presence of ``@dataclass_transform()`` tells a static type checker that the + decorated object performs runtime "magic" that + transforms a class, giving it :func:`dataclasses.dataclass`-like behaviors. + + Example usage with a decorator function:: + + T = TypeVar("T") + + @dataclass_transform() + def create_model(cls: type[T]) -> type[T]: + ... + return cls + + @create_model + class CustomerModel: + id: int + name: str + + On a base class:: + + @dataclass_transform() + class ModelBase: ... + + class CustomerModel(ModelBase): + id: int + name: str + + On a metaclass:: + + @dataclass_transform() + class ModelMeta(type): ... + + class ModelBase(metaclass=ModelMeta): ... + + class CustomerModel(ModelBase): + id: int + name: str + + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + :func:`@dataclasses.dataclass `. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. + + The arguments to this decorator can be used to customize this behavior: + + * ``eq_default`` indicates whether the ``eq`` parameter is assumed to be + ``True`` or ``False`` if it is omitted by the caller. + * ``order_default`` indicates whether the ``order`` parameter is + assumed to be True or False if it is omitted by the caller. + * ``kw_only_default`` indicates whether the ``kw_only`` parameter is + assumed to be True or False if it is omitted by the caller. + * ``field_specifiers`` specifies a static list of supported classes + or functions that describe fields, similar to ``dataclasses.field()``. + * Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. + + At runtime, this decorator records its arguments in the + ``__dataclass_transform__`` attribute on the decorated object. + It has no other runtime effect. + + See :pep:`681` for more details. + + .. versionadded:: 3.11 + .. decorator:: overload The ``@overload`` decorator allows describing functions and methods @@ -2668,7 +2789,7 @@ Constant .. note:: - If ``from __future__ import annotations`` is used in Python 3.7 or later, + If ``from __future__ import annotations`` is used, annotations are not evaluated at function definition time. Instead, they are stored as strings in ``__annotations__``. This makes it unnecessary to use quotes around the annotation diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 24a18c6848468..054efa8126632 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -660,7 +660,7 @@ Applying the same patch to every test method If you want several patches in place for multiple test methods the obvious way is to apply the patch decorators to every method. This can feel like unnecessary -repetition. For Python 2.6 or more recent you can use :func:`patch` (in all its +repetition. Instead, you can use :func:`patch` (in all its various forms) as a class decorator. This applies the patches to all test methods on the class. A test method is identified by methods whose names start with ``test``:: diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index a3700ac07f1c1..acc0d67541ae8 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2381,7 +2381,7 @@ FILTER_DIR .. data:: FILTER_DIR :data:`FILTER_DIR` is a module level variable that controls the way mock objects -respond to :func:`dir` (only for Python 2.6 or more recent). The default is ``True``, +respond to :func:`dir`. The default is ``True``, which uses the filtering described below, to only show useful members. If you dislike this filtering, or need to switch it off for diagnostic purposes, then set ``mock.FILTER_DIR = False``. diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 9b8b75acce514..e07a32b88b1c3 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1495,6 +1495,16 @@ Test cases .. versionadded:: 3.1 + .. method:: enterContext(cm) + + Enter the supplied :term:`context manager`. If successful, also + add its :meth:`~object.__exit__` method as a cleanup function by + :meth:`addCleanup` and return the result of the + :meth:`~object.__enter__` method. + + .. versionadded:: 3.11 + + .. method:: doCleanups() This method is called unconditionally after :meth:`tearDown`, or @@ -1510,6 +1520,7 @@ Test cases .. versionadded:: 3.1 + .. classmethod:: addClassCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDownClass` to cleanup @@ -1524,6 +1535,16 @@ Test cases .. versionadded:: 3.8 + .. classmethod:: enterClassContext(cm) + + Enter the supplied :term:`context manager`. If successful, also + add its :meth:`~object.__exit__` method as a cleanup function by + :meth:`addClassCleanup` and return the result of the + :meth:`~object.__enter__` method. + + .. versionadded:: 3.11 + + .. classmethod:: doClassCleanups() This method is called unconditionally after :meth:`tearDownClass`, or @@ -1571,6 +1592,16 @@ Test cases This method accepts a coroutine that can be used as a cleanup function. + .. coroutinemethod:: enterAsyncContext(cm) + + Enter the supplied :term:`asynchronous context manager`. If successful, + also add its :meth:`~object.__aexit__` method as a cleanup function by + :meth:`addAsyncCleanup` and return the result of the + :meth:`~object.__aenter__` method. + + .. versionadded:: 3.11 + + .. method:: run(result=None) Sets up a new event loop to run the test, collecting the result into @@ -2465,13 +2496,23 @@ To add cleanup code that must be run even in the case of an exception, use .. versionadded:: 3.8 +.. classmethod:: enterModuleContext(cm) + + Enter the supplied :term:`context manager`. If successful, also + add its :meth:`~object.__exit__` method as a cleanup function by + :func:`addModuleCleanup` and return the result of the + :meth:`~object.__enter__` method. + + .. versionadded:: 3.11 + + .. function:: doModuleCleanups() This function is called unconditionally after :func:`tearDownModule`, or after :func:`setUpModule` if :func:`setUpModule` raises an exception. It is responsible for calling all the cleanup functions added by - :func:`addCleanupModule`. If you need cleanup functions to be called + :func:`addModuleCleanup`. If you need cleanup functions to be called *prior* to :func:`tearDownModule` then you can call :func:`doModuleCleanups` yourself. @@ -2480,6 +2521,7 @@ To add cleanup code that must be run even in the case of an exception, use .. versionadded:: 3.8 + Signal Handling --------------- diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst index c341bc83dcfed..83c4aec47bbef 100644 --- a/Doc/library/uu.rst +++ b/Doc/library/uu.rst @@ -9,8 +9,10 @@ **Source code:** :source:`Lib/uu.py` -.. deprecated:: 3.11 - The :mod:`uu` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`uu` module is deprecated + (see :pep:`PEP 594 <594#uu-and-the-uu-encoding>` for details). + :mod:`base64` is a modern alternative. -------------- diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index f7a1f70833b7f..43708f8021ad2 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -154,14 +154,19 @@ the disposition of the match. Each entry is a tuple of the form (*action*, +---------------+----------------------------------------------+ * *message* is a string containing a regular expression that the start of - the warning message must match. The expression is compiled to always be - case-insensitive. + the warning message must match, case-insensitively. In :option:`-W` and + :envvar:`PYTHONWARNINGS`, *message* is a literal string that the start of the + warning message must contain (case-insensitively), ignoring any whitespace at + the start or end of *message*. * *category* is a class (a subclass of :exc:`Warning`) of which the warning category must be a subclass in order to match. -* *module* is a string containing a regular expression that the module name must - match. The expression is compiled to be case-sensitive. +* *module* is a string containing a regular expression that the start of the + fully-qualified module name must match, case-sensitively. In :option:`-W` and + :envvar:`PYTHONWARNINGS`, *module* is a literal string that the + fully-qualified module name must be equal to (case-sensitively), ignoring any + whitespace at the start or end of *module*. * *lineno* is an integer that the line number where the warning occurred must match, or ``0`` to match all line numbers. @@ -207,8 +212,7 @@ Some examples:: error::ResourceWarning # Treat ResourceWarning messages as errors default::DeprecationWarning # Show DeprecationWarning messages ignore,default:::mymodule # Only report warnings triggered by "mymodule" - error:::mymodule[.*] # Convert warnings to errors in "mymodule" - # and any subpackages of "mymodule" + error:::mymodule # Convert warnings to errors in "mymodule" .. _default-warning-filter: diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 1102c634edaf3..8397de4fb488f 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -1,3 +1,5 @@ +.. _mod-weakref: + :mod:`weakref` --- Weak references ================================== diff --git a/Doc/library/xdrlib.rst b/Doc/library/xdrlib.rst index 060b2e2c60df6..39e75573260c5 100644 --- a/Doc/library/xdrlib.rst +++ b/Doc/library/xdrlib.rst @@ -11,8 +11,9 @@ single: XDR single: External Data Representation -.. deprecated:: 3.11 - The :mod:`xdrlib` module is deprecated (see :pep:`594` for details). +.. deprecated-removed:: 3.11 3.13 + The :mod:`xdrlib` module is deprecated + (see :pep:`PEP 594 <594#xdrlib>` for details). -------------- diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index e3932bc9e659f..2fe0d2e082fb3 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -826,6 +826,7 @@ Functions ^^^^^^^^^ .. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) + :module: Default loader. This default loader reads an included resource from disk. *href* is a URL. *parse* is for parse mode either "xml" or "text". *encoding* @@ -837,6 +838,7 @@ Functions .. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ max_depth=6) + :module: This function expands XInclude directives. *elem* is the root element. *loader* is an optional resource loader. If omitted, it defaults to :func:`default_loader`. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 19e128ce02f54..4dd9fa961a8d9 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -139,7 +139,7 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ - compresslevel=None, *, strict_timestamps=True, + compresslevel=None, *, strict_timestamps=True, \ metadata_encoding=None) Open a ZIP file, where *file* can be a path to a file (a string), a diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index f0c67d5ae2584..30eb99817a859 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -43,8 +43,6 @@ The available exception and functions in this module are: .. versionchanged:: 3.0 The result is always unsigned. - To generate the same numeric value when using Python 2 or earlier, - use ``adler32(data) & 0xffffffff``. .. function:: compress(data, /, level=-1, wbits=MAX_WBITS) @@ -137,8 +135,6 @@ The available exception and functions in this module are: .. versionchanged:: 3.0 The result is always unsigned. - To generate the same numeric value when using Python 2 or earlier, - use ``crc32(data) & 0xffffffff``. .. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) diff --git a/Doc/make.bat b/Doc/make.bat index d9a7aa4ca7fa6..4f0b3c11f4fac 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -180,7 +180,10 @@ if EXIST "%BUILDDIR%\html\index.html" ( goto end :check -cmd /S /C "%SPHINXLINT% -i tools" +rem Check the docs and NEWS files with sphinx-lint. +rem Ignore the tools dir and check that the default role is not used. +cmd /S /C "%SPHINXLINT% -i tools --enable default-role" +cmd /S /C "%SPHINXLINT% --enable default-role ..\Misc\NEWS.d\next\ " goto end :serve diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b914c48d3d4cd..1f461e3fed8e5 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1898,7 +1898,7 @@ precedence and have a left-to-right chaining feature as described in the | ``x[index]``, ``x[index:index]``, | Subscription, slicing, | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ -| :keyword:`await` ``x`` | Await expression | +| :keyword:`await x ` | Await expression | +-----------------------------------------------+-------------------------------------+ | ``**`` | Exponentiation [#]_ | +-----------------------------------------------+-------------------------------------+ @@ -1922,7 +1922,7 @@ precedence and have a left-to-right chaining feature as described in the | :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | | ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +-----------------------------------------------+-------------------------------------+ -| :keyword:`not` ``x`` | Boolean NOT | +| :keyword:`not x ` | Boolean NOT | +-----------------------------------------------+-------------------------------------+ | :keyword:`and` | Boolean AND | +-----------------------------------------------+-------------------------------------+ diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index b02e31dc88e5c..bdcfe9ac1c23d 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -429,7 +429,8 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)', re.I) +gh_issue_re = re.compile('(?:gh-issue-|gh-)([0-9]+)', re.I) whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -456,9 +457,9 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`bpo-\1 `__', - content) + content = issue_re.sub(r':issue:`\1`', content) + # Fallback handling for the GitHub issue + content = gh_issue_re.sub(r':gh:`\1`', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index f44cb0b4e905a..58b06eb5f2535 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -479,9 +479,9 @@ If the same attribute name occurs in both an instance and in a class, then attribute lookup prioritizes the instance:: >>> class Warehouse: - purpose = 'storage' - region = 'west' - + ... purpose = 'storage' + ... region = 'west' + ... >>> w1 = Warehouse() >>> print(w1.purpose, w1.region) storage west diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 524d3c32417b9..d6f2464ae0829 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,8 @@ The Module Search Path .. index:: triple: module; search; path When a module named :mod:`spam` is imported, the interpreter first searches for -a built-in module with that name. If not found, it then searches for a file +a built-in module with that name. These module names are listed in +:data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable :data:`sys.path`. :data:`sys.path` is initialized from these locations: diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 2f132a96bef08..f7db038430b6d 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -160,7 +160,7 @@ Distributing Python Applications on the Mac The standard tool for deploying standalone Python applications on the Mac is :program:`py2app`. More information on installing and using py2app can be found -at http://undefined.org/python/#py2app. +at https://pypi.org/project/py2app/. Other Resources diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 88dcb002e2c24..9c2632f030bdc 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -378,7 +378,9 @@ may be changed from ``.``, and the package will be installed into a subdirectory. By default, the subdirectory is named the same as the package, and without the ``-ExcludeVersion`` option this name will include the specific version installed. Inside the subdirectory is a ``tools`` directory that -contains the Python installation:: +contains the Python installation: + +.. code-block:: doscon # Without -ExcludeVersion > .\python.3.5.2\tools\python.exe -V @@ -425,7 +427,7 @@ dependants, such as Idle), pip and the Python documentation are not included. .. note:: The embedded distribution does not include the `Microsoft C Runtime - `_ and it is + `_ and it is the responsibility of the application installer to provide this. The runtime may have already been installed on a user's system previously or automatically via Windows Update, and can be detected by finding @@ -559,27 +561,22 @@ System variables, you need non-restricted access to your machine Windows will concatenate User variables *after* System variables, which may cause unexpected results when modifying :envvar:`PATH`. - The :envvar:`PYTHONPATH` variable is used by all versions of Python 2 and - Python 3, so you should not permanently configure this variable unless it - only includes code that is compatible with all of your installed Python + The :envvar:`PYTHONPATH` variable is used by all versions of Python, + so you should not permanently configure it unless the listed paths + only include code that is compatible with all of your installed Python versions. .. seealso:: - https://www.microsoft.com/en-us/wdsi/help/folder-variables - Environment variables in Windows NT - - https://technet.microsoft.com/en-us/library/cc754250.aspx - The SET command, for temporarily modifying environment variables + https://docs.microsoft.com/en-us/windows/win32/procthread/environment-variables + Overview of environment variables on Windows - https://technet.microsoft.com/en-us/library/cc755104.aspx - The SETX command, for permanently modifying environment variables + https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/set_1 + The ``set`` command, for temporarily modifying environment variables - https://support.microsoft.com/en-us/help/310519/how-to-manage-environment-variables-in-windows-xp - How To Manage Environment Variables in Windows XP + https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/setx + The ``setx`` command, for permanently modifying environment variables - https://www.chem.gla.ac.uk/~louis/software/faq/q1.html - Setting Environment variables, Louis J. Farrugia .. _windows-path-mod: @@ -677,9 +674,7 @@ From the command-line System-wide installations of Python 3.3 and later will put the launcher on your :envvar:`PATH`. The launcher is compatible with all available versions of Python, so it does not matter which version is installed. To check that the -launcher is available, execute the following command in Command Prompt: - -:: +launcher is available, execute the following command in Command Prompt:: py @@ -687,26 +682,20 @@ You should find that the latest version of Python you have installed is started - it can be exited as normal, and any additional command-line arguments specified will be sent directly to Python. -If you have multiple versions of Python installed (e.g., 2.7 and |version|) you -will have noticed that Python |version| was started - to launch Python 2.7, try -the command: - -:: - - py -2.7 +If you have multiple versions of Python installed (e.g., 3.7 and |version|) you +will have noticed that Python |version| was started - to launch Python 3.7, try +the command:: -If you want the latest version of Python 2.x you have installed, try the -command: + py -3.7 -:: +If you want the latest version of Python 2 you have installed, try the +command:: py -2 -You should find the latest version of Python 2.x starts. +You should find the latest version of Python 3.x starts. -If you see the following error, you do not have the launcher installed: - -:: +If you see the following error, you do not have the launcher installed:: 'py' is not recognized as an internal or external command, operable program or batch file. @@ -714,11 +703,11 @@ If you see the following error, you do not have the launcher installed: Per-user installations of Python do not add the launcher to :envvar:`PATH` unless the option was selected on installation. -:: +The command:: py --list -You should see the currently installed versions of Python. +displays the currently installed version(s) of Python. Virtual environments ^^^^^^^^^^^^^^^^^^^^ @@ -744,9 +733,7 @@ following contents import sys sys.stdout.write("hello from Python %s\n" % (sys.version,)) -From the directory in which hello.py lives, execute the command: - -:: +From the directory in which hello.py lives, execute the command:: py hello.py @@ -759,9 +746,9 @@ is printed. Now try changing the first line to be: Re-executing the command should now print the latest Python 3.x information. As with the above command-line examples, you can specify a more explicit -version qualifier. Assuming you have Python 2.6 installed, try changing the -first line to ``#! python2.6`` and you should find the 2.6 version -information printed. +version qualifier. Assuming you have Python 3.7 installed, try changing +the first line to ``#! python3.7`` and you should find the |version| +version information printed. Note that unlike interactive use, a bare "python" will use the latest version of Python 2.x that you have installed. This is for backward @@ -814,8 +801,8 @@ shebang lines starting with ``/usr``. Any of the above virtual commands can be suffixed with an explicit version (either just the major version, or the major and minor version). Furthermore the 32-bit version can be requested by adding "-32" after the -minor version. I.e. ``/usr/bin/python2.7-32`` will request usage of the -32-bit python 2.7. +minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the +32-bit python 3.7. .. versionadded:: 3.7 @@ -908,19 +895,19 @@ Examples: ``python2`` will use the latest Python 2.x version installed and the command ``python3`` will use the latest Python 3.x installed. -* The commands ``python3.1`` and ``python2.7`` will not consult any +* The command ``python3.7`` will not consult any options at all as the versions are fully specified. * If ``PY_PYTHON=3``, the commands ``python`` and ``python3`` will both use the latest installed Python 3 version. -* If ``PY_PYTHON=3.1-32``, the command ``python`` will use the 32-bit - implementation of 3.1 whereas the command ``python3`` will use the latest +* If ``PY_PYTHON=3.7-32``, the command ``python`` will use the 32-bit + implementation of 3.7 whereas the command ``python3`` will use the latest installed Python (PY_PYTHON was not considered at all as a major version was specified.) -* If ``PY_PYTHON=3`` and ``PY_PYTHON3=3.1``, the commands - ``python`` and ``python3`` will both use specifically 3.1 +* If ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7``, the commands + ``python`` and ``python3`` will both use specifically 3.7 In addition to environment variables, the same settings can be configured in the .INI file used by the launcher. The section in the INI file is @@ -931,21 +918,21 @@ an environment variable will override things specified in the INI file. For example: -* Setting ``PY_PYTHON=3.1`` is equivalent to the INI file containing: +* Setting ``PY_PYTHON=3.7`` is equivalent to the INI file containing: .. code-block:: ini [defaults] - python=3.1 + python=3.7 -* Setting ``PY_PYTHON=3`` and ``PY_PYTHON3=3.1`` is equivalent to the INI file +* Setting ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7`` is equivalent to the INI file containing: .. code-block:: ini [defaults] python=3 - python3=3.1 + python3=3.7 Diagnostics ----------- @@ -1132,13 +1119,14 @@ is a collection of modules for advanced Windows-specific support. This includes utilities for: * `Component Object Model - `_ + `_ (COM) * Win32 API calls * Registry * Event log -* `Microsoft Foundation Classes `_ (MFC) - user interfaces +* `Microsoft Foundation Classes + `_ + (MFC) user interfaces `PythonWin `_ is a sample MFC application @@ -1149,7 +1137,7 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. `Win32 How Do I...? `_ by Tim Golden - `Python and COM `_ + `Python and COM `_ by David and Paul Boddie @@ -1163,18 +1151,6 @@ you can distribute your application without requiring your users to install Python. -WConio ------- - -Since Python's advanced terminal handling layer, :mod:`curses`, is restricted to -Unix-like systems, there is a library exclusive to Windows as well: Windows -Console I/O for Python. - -`WConio `_ is a wrapper for -Turbo-C's :file:`CONIO.H`, used to create text user interfaces. - - - Compiling Python on Windows =========================== @@ -1184,21 +1160,13 @@ latest release's source or just grab a fresh `checkout `_. The source tree contains a build solution and project files for Microsoft -Visual Studio 2015, which is the compiler used to build the official Python +Visual Studio, which is the compiler used to build the official Python releases. These files are in the :file:`PCbuild` directory. Check :file:`PCbuild/readme.txt` for general information on the build process. - For extension modules, consult :ref:`building-on-windows`. -.. seealso:: - - `Python + Windows + distutils + SWIG + gcc MinGW `_ - or "Creating Python extensions in C/C++ with SWIG and compiling them with - MinGW gcc under Windows" or "Installing Python extension with distutils - and without Microsoft Visual C++" by Sébastien Sauvage, 2003 - Other Platforms =============== @@ -1207,12 +1175,12 @@ With ongoing development of Python, some platforms that used to be supported earlier are no longer supported (due to the lack of users or developers). Check :pep:`11` for details on all unsupported platforms. -* `Windows CE `_ is still supported. -* The `Cygwin `_ installer offers to install the Python - interpreter as well (cf. `Cygwin package source - `_, `Maintainer releases - `_) +* `Windows CE `_ is + `no longer supported `__ + since Python 3 (if it ever was). +* The `Cygwin `_ installer offers to install the + `Python interpreter `__ + as well See `Python for Windows `_ for detailed information about platforms with pre-compiled installers. diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 4e85abaea7553..103a72f913c63 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -11,7 +11,7 @@ This article explains the new features in Python 2.5. The final release of Python 2.5 is scheduled for August 2006; :pep:`356` describes the planned -release schedule. +release schedule. Python 2.5 was released on September 19, 2006. The changes in Python 2.5 are an interesting mix of language and library improvements. The library enhancements will be more important to Python's user @@ -1767,7 +1767,7 @@ included. The rest of this section will provide a brief overview of using ElementTree. Full documentation for ElementTree is available at -http://effbot.org/zone/element-index.htm. +https://web.archive.org/web/20201124024954/http://effbot.org/zone/element-index.htm. ElementTree represents an XML document as a tree of element nodes. The text content of the document is stored as the :attr:`text` and :attr:`tail` @@ -1865,7 +1865,7 @@ read the package's official documentation for more details. .. seealso:: - http://effbot.org/zone/element-index.htm + https://web.archive.org/web/20201124024954/http://effbot.org/zone/element-index.htm Official documentation for ElementTree. .. ====================================================================== diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index b6174a19a178b..08a7c58b0f080 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -49,7 +49,7 @@ This saves the maintainer some effort going through the SVN logs when researching a change. -This article explains the new features in Python 2.6, released on October 1 +This article explains the new features in Python 2.6, released on October 1, 2008. The release schedule is described in :pep:`361`. The major theme of Python 2.6 is preparing the migration path to diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 999f148fa6b79..297ce0647130e 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2089,7 +2089,7 @@ version 1.3. Some of the new features are: Fredrik Lundh develops ElementTree and produced the 1.3 version; you can read his article describing 1.3 at -http://effbot.org/zone/elementtree-13-intro.htm. +https://web.archive.org/web/20200703234532/http://effbot.org/zone/elementtree-13-intro.htm. Florent Xicluna updated the version included with Python, after discussions on python-dev and in :issue:`6472`.) diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 880958d3edb90..4da3507ad2e89 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -53,9 +53,9 @@ This article explains the new features in Python 3.0, compared to 2.6. Python 3.0, also known as "Python 3000" or "Py3K", is the first ever -*intentionally backwards incompatible* Python release. There are more -changes than in a typical release, and more that are important for all -Python users. Nevertheless, after digesting the changes, you'll find +*intentionally backwards incompatible* Python release. Python 3.0 was released on December 3, 2008. +There are more changes than in a typical release, and more that are important for all +Python users. Nevertheless, after digesting the changes, you'll find that Python really hasn't changed all that much -- by and large, we're mostly fixing well-known annoyances and warts, and removing a lot of old cruft. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index f1e6d0c4f3dd6..3d89b97fa8f1b 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -47,6 +47,7 @@ when researching a change. This article explains the new features in Python 3.1, compared to 3.0. +Python 3.1 was released on June 27, 2009. PEP 372: Ordered Dictionaries diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f93523a4c50a8..1f2e2a81860c1 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -47,7 +47,7 @@ when researching a change. This article explains the new features in Python 3.10, compared to 3.9. - +Python 3.10 was released on October 4, 2021. For full details, see the :ref:`changelog `. Summary -- Release highlights diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index b9871f7ef69f9..4f9bafefe2b9b 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -63,7 +63,7 @@ Summary -- Release highlights Brevity is key. - Python 3.11 is up to 10-60% faster than Python 3.10. On average, we measured a - 1.22x speedup on the standard benchmark suite. See `Faster CPython`_ for details. + 1.25x speedup on the standard benchmark suite. See `Faster CPython`_ for details. .. PEP-sized items next. @@ -306,23 +306,23 @@ Kumar Srinivasan and Graham Bleaney.) PEP 681: Data Class Transforms ------------------------------ -The new :data:`~typing.dataclass_transform` annotation may be used to -decorate a function that is itself a decorator, a class, or a metaclass. +:data:`~typing.dataclass_transform` may be used to +decorate a class, metaclass, or a function that is itself a decorator. The presence of ``@dataclass_transform()`` tells a static type checker that the -decorated function, class, or metaclass performs runtime "magic" that -transforms a class, endowing it with dataclass-like behaviors. +decorated object performs runtime "magic" that +transforms a class, giving it :func:`dataclasses.dataclass`-like behaviors. For example:: - # The ``create_model`` decorator is defined by a library. + # The create_model decorator is defined by a library. @typing.dataclass_transform() - def create_model(cls: Type[_T]) -> Type[_T]: + def create_model(cls: Type[T]) -> Type[T]: cls.__init__ = ... cls.__eq__ = ... cls.__ne__ = ... return cls - # The ``create_model`` decorator can now be used to create new model + # The create_model decorator can now be used to create new model # classes, like this: @create_model class CustomerModel: @@ -403,6 +403,11 @@ Other CPython Implementation Changes instead of prepending them. (Contributed by Bastian Neuburger in :issue:`44934`.) +* The :c:member:`PyConfig.module_search_paths_set` field must now be set to 1 for + initialization to use :c:member:`PyConfig.module_search_paths` to initialize + :data:`sys.path`. Otherwise, initialization will recalculate the path and replace + any values added to ``module_search_paths``. + New Modules =========== @@ -431,9 +436,15 @@ asyncio existing stream-based connections to TLS. (Contributed by Ian Good in :issue:`34975`.) +* Add :class:`~asyncio.Barrier` class to the synchronization primitives of + the asyncio library. (Contributed by Yves Duprat and Andrew Svetlov in + :gh:`87518`.) + datetime -------- +* Add :attr:`datetime.UTC`, a convenience alias for + :attr:`datetime.timezone.utc`. (Contributed by Kabir Kwatra in :gh:`91973`.) * :meth:`datetime.date.fromisoformat`, :meth:`datetime.time.fromisoformat` and :meth:`datetime.datetime.fromisoformat` can now be used to parse most ISO 8601 formats (barring only those that support fractional hours and minutes). @@ -724,6 +735,12 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. the given type. At runtime it simply returns the received value. (Contributed by Jelle Zijlstra in :gh:`90638`.) +* :data:`typing.TypedDict` types can now be generic. (Contributed by + Samodya Abey in :gh:`89026`.) + +* :class:`~typing.NamedTuple` types can now be generic. + (Contributed by Serhiy Storchaka in :issue:`43923`.) + * Allow subclassing of :class:`typing.Any`. This is useful for avoiding type checker errors related to highly dynamic class, such as mocks. (Contributed by Shantanu Jain in :gh:`91154`.) @@ -737,11 +754,33 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. to clear all registered overloads of a function. (Contributed by Jelle Zijlstra in :gh:`89263`.) -* :data:`typing.TypedDict` subclasses can now be generic. (Contributed by - Samodya Abey in :gh:`89026`.) +* The :meth:`__init__` method of :class:`~typing.Protocol` subclasses + is now preserved. (Contributed by Adrian Garcia Badarasco in :gh:`88970`.) -* :class:`~typing.NamedTuple` subclasses can now be generic. - (Contributed by Serhiy Storchaka in :issue:`43923`.) +* The representation of empty tuple types (``Tuple[()]``) is simplified. + This affects introspection, e.g. ``get_args(Tuple[()])`` now evaluates + to ``()`` instead of ``((),)``. + (Contributed by Serhiy Storchaka in :gh:`91137`.) + +* Loosen runtime requirements for type annotations by removing the callable + check in the private ``typing._type_check`` function. (Contributed by + Gregory Beauregard in :gh:`90802`.) + +* :func:`typing.get_type_hints` now supports evaluating strings as forward + references in :ref:`PEP 585 generic aliases `. + (Contributed by Niklas Rosenstein in :gh:`85542`.) + +* :func:`typing.get_type_hints` no longer adds :data:`~typing.Optional` + to parameters with ``None`` as a default. (Contributed by Nikita Sobolev + in :gh:`90353`.) + +* :func:`typing.get_type_hints` now supports evaluating bare stringified + :data:`~typing.ClassVar` annotations. (Contributed by Gregory Beauregard + in :gh:`90711`.) + +* :func:`typing.no_type_check` no longer modifies external classes and functions. + It also now correctly marks classmethods as not to be type checked. (Contributed + by Nikita Sobolev in :gh:`90729`.) tkinter @@ -758,6 +797,18 @@ unicodedata * The Unicode database has been updated to version 14.0.0. (:issue:`45190`). +unittest +-------- + +* Added methods :meth:`~unittest.TestCase.enterContext` and + :meth:`~unittest.TestCase.enterClassContext` of class + :class:`~unittest.TestCase`, method + :meth:`~unittest.IsolatedAsyncioTestCase.enterAsyncContext` of + class :class:`~unittest.IsolatedAsyncioTestCase` and function + :func:`unittest.enterModuleContext`. + (Contributed by Serhiy Storchaka in :issue:`45046`.) + + venv ---- @@ -828,7 +879,7 @@ Optimizations Faster CPython ============== -CPython 3.11 is on average `1.22x faster `_ +CPython 3.11 is on average `25% faster `_ than CPython 3.10 when measured with the `pyperformance `_ benchmark suite, and compiled with GCC on Ubuntu Linux. Depending on your workload, the speedup @@ -879,10 +930,11 @@ holds execution information. The following are new frame optimizations: - Streamlined the internal frame struct to contain only essential information. Frames previously held extra debugging and memory management information. -Old-style frame objects are now created only when required by debuggers. For -most user code, no frame objects are created at all. As a result, nearly all -Python functions calls have sped up significantly. We measured a 3-7% speedup -in pyperformance. +Old-style frame objects are now created only when requested by debuggers or +by Python introspection functions such as ``sys._getframe`` or +``inspect.currentframe``. For most user code, no frame objects are +created at all. As a result, nearly all Python functions calls have sped +up significantly. We measured a 3-7% speedup in pyperformance. (Contributed by Mark Shannon in :issue:`44590`.) @@ -930,7 +982,8 @@ and specialization attempts are not too expensive. This allows specialization to adapt to new circumstances. (PEP written by Mark Shannon, with ideas inspired by Stefan Brunthaler. -See :pep:`659` for more information.) +See :pep:`659` for more information. Implementation by Mark Shannon and Brandt +Bucher, with additional help from Irit Katriel and Dennis Sweeney.) .. If I missed out anyone, please add them. @@ -1100,8 +1153,8 @@ Deprecated that was added in Python 3.10. (Contributed by Raymond Hettinger in :gh:`89519`.) -* Octal escapes with value larger than ``0o377`` now produce - a :exc:`DeprecationWarning`. +* Octal escapes in string and bytes literals with value larger than ``0o377`` now + produce :exc:`DeprecationWarning`. In a future Python version they will be a :exc:`SyntaxWarning` and eventually a :exc:`SyntaxError`. (Contributed by Serhiy Storchaka in :gh:`81548`.) @@ -1154,7 +1207,7 @@ Deprecated * the :class:`configparser.SafeConfigParser` class * the :attr:`configparser.ParsingError.filename` property - * the :meth:`configparser.ParsingError.readfp` method + * the :meth:`configparser.RawConfigParser.readfp` method (Contributed by Hugo van Kemenade in :issue:`45173`.) @@ -1168,7 +1221,11 @@ Deprecated removed in Python 3.13. Use :func:`locale.setlocale`, :func:`locale.getpreferredencoding(False) ` and :func:`locale.getlocale` functions instead. - (Contributed by Victor Stinner in :issue:`46659`.) + (Contributed by Victor Stinner in :gh:`90817`.) + +* The :func:`locale.resetlocale` function is deprecated and will be + removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`90817`.) * The :mod:`asynchat`, :mod:`asyncore` and :mod:`smtpd` modules have been deprecated since at least Python 3.6. Their documentation and deprecation @@ -1216,6 +1273,70 @@ Deprecated wherever possible. (Contributed by Alex Waygood in :gh:`92332`.) +* The keyword argument syntax for constructing :data:`~typing.TypedDict` types + is now deprecated. Support will be removed in Python 3.13. (Contributed by + Jingchen Ye in :gh:`90224`.) + +* The :func:`re.template` function and the corresponding :const:`re.TEMPLATE` + and :const:`re.T` flags are deprecated, as they were undocumented and + lacked an obvious purpose. They will be removed in Python 3.13. + (Contributed by Serhiy Storchaka and Miro Hrončok in :gh:`92728`.) + + +Pending Removal in Python 3.12 +============================== + +The following APIs have been deprecated in earlier Python releases, +and will be removed in Python 3.12. + +Python API: + +* :class:`pkgutil.ImpImporter` +* :class:`pkgutil.ImpLoader` +* :envvar:`PYTHONTHREADDEBUG` +* :func:`importlib.find_loader` +* :func:`importlib.util.module_for_loader` +* :func:`importlib.util.set_loader_wrapper` +* :func:`importlib.util.set_package_wrapper` +* :meth:`importlib.abc.Loader.module_repr` +* :meth:`importlib.abc.Loadermodule_repr` +* :meth:`importlib.abc.MetaPathFinder.find_module` +* :meth:`importlib.abc.MetaPathFinder.find_module` +* :meth:`importlib.abc.PathEntryFinder.find_loader` +* :meth:`importlib.abc.PathEntryFinder.find_module` +* :meth:`importlib.machinery.BuiltinImporter.find_module` +* :meth:`importlib.machinery.BuiltinLoader.module_repr` +* :meth:`importlib.machinery.FileFinder.find_loader` +* :meth:`importlib.machinery.FileFinder.find_module` +* :meth:`importlib.machinery.FrozenImporter.find_module` +* :meth:`importlib.machinery.FrozenLoader.module_repr` +* :meth:`importlib.machinery.PathFinder.find_module` +* :meth:`importlib.machinery.WindowsRegistryFinder.find_module` +* :meth:`pathlib.Path.link_to` +* The entire :ref:`distutils namespace ` +* :func:`cgi.log` +* :func:`sqlite3.OptimizedUnicode` +* :func:`sqlite3.enable_shared_cache` + +C API: + +* :c:func:`PyUnicode_AS_DATA` +* :c:func:`PyUnicode_AS_UNICODE` +* :c:func:`PyUnicode_AsUnicodeAndSize` +* :c:func:`PyUnicode_AsUnicode` +* :c:func:`PyUnicode_FromUnicode` +* :c:func:`PyUnicode_GET_DATA_SIZE` +* :c:func:`PyUnicode_GET_SIZE` +* :c:func:`PyUnicode_GetSize` +* :c:func:`PyUnicode_IS_COMPACT` +* :c:func:`PyUnicode_IS_READY` +* :c:func:`PyUnicode_READY` +* :c:func:`Py_UNICODE_WSTR_LENGTH` +* :c:func:`_PyUnicode_AsUnicode` +* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:type:`PyUnicodeObject` +* :c:func:`PyUnicode_InternImmortal()` + Removed ======= @@ -1304,6 +1425,11 @@ Removed Python's test suite." (Contributed by Victor Stinner in :issue:`46852`.) +* The ``--experimental-isolated-subinterpreters`` configure flag + (and corresponding ``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS``) + have been removed. + + Porting to Python 3.11 ====================== @@ -1433,37 +1559,6 @@ Build Changes C API Changes ============= -* :c:func:`PyErr_SetExcInfo()` no longer uses the ``type`` and ``traceback`` - arguments, the interpreter now derives those values from the exception - instance (the ``value`` argument). The function still steals references - of all three arguments. - (Contributed by Irit Katriel in :issue:`45711`.) - -* :c:func:`PyErr_GetExcInfo()` now derives the ``type`` and ``traceback`` - fields of the result from the exception instance (the ``value`` field). - (Contributed by Irit Katriel in :issue:`45711`.) - -* :c:type:`_frozen` has a new ``is_package`` field to indicate whether - or not the frozen module is a package. Previously, a negative value - in the ``size`` field was the indicator. Now only non-negative values - be used for ``size``. - (Contributed by Kumar Aditya in :issue:`46608`.) - -* :c:func:`_PyFrameEvalFunction` now takes ``_PyInterpreterFrame*`` - as its second parameter, instead of ``PyFrameObject*``. - See :pep:`523` for more details of how to use this function pointer type. - -* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take - an additional ``exception_table`` argument. - Using these functions should be avoided, if at all possible. - To get a custom code object: create a code object using the compiler, - then get a modified version with the ``replace`` method. - -* :c:type:`PyCodeObject` no longer has a ``co_code`` field. Instead, - use ``PyObject_GetAttrString(code_object, "co_code")`` or - :c:func:`PyCode_GetCode` to get the underlying bytes object. - (Contributed by Brandt Bucher in :issue:`46841` and Ken Jin in :gh:`92154`.) - New Features ------------ @@ -1530,6 +1625,37 @@ New Features Porting to Python 3.11 ---------------------- +* :c:func:`PyErr_SetExcInfo()` no longer uses the ``type`` and ``traceback`` + arguments, the interpreter now derives those values from the exception + instance (the ``value`` argument). The function still steals references + of all three arguments. + (Contributed by Irit Katriel in :issue:`45711`.) + +* :c:func:`PyErr_GetExcInfo()` now derives the ``type`` and ``traceback`` + fields of the result from the exception instance (the ``value`` field). + (Contributed by Irit Katriel in :issue:`45711`.) + +* :c:type:`_frozen` has a new ``is_package`` field to indicate whether + or not the frozen module is a package. Previously, a negative value + in the ``size`` field was the indicator. Now only non-negative values + be used for ``size``. + (Contributed by Kumar Aditya in :issue:`46608`.) + +* :c:func:`_PyFrameEvalFunction` now takes ``_PyInterpreterFrame*`` + as its second parameter, instead of ``PyFrameObject*``. + See :pep:`523` for more details of how to use this function pointer type. + +* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take + an additional ``exception_table`` argument. + Using these functions should be avoided, if at all possible. + To get a custom code object: create a code object using the compiler, + then get a modified version with the ``replace`` method. + +* :c:type:`PyCodeObject` no longer has a ``co_code`` field. Instead, + use ``PyObject_GetAttrString(code_object, "co_code")`` or + :c:func:`PyCode_GetCode` to get the underlying bytes object. + (Contributed by Brandt Bucher in :issue:`46841` and Ken Jin in :gh:`92154`.) + * The old trashcan macros (``Py_TRASHCAN_SAFE_BEGIN``/``Py_TRASHCAN_SAFE_END``) are now deprecated. They should be replaced by the new macros ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. @@ -1754,6 +1880,16 @@ Porting to Python 3.11 * Distributors are encouraged to build Python with the optimized Blake2 library `libb2`_. +* The :c:member:`PyConfig.module_search_paths_set` field must now be set to 1 for + initialization to use :c:member:`PyConfig.module_search_paths` to initialize + :data:`sys.path`. Otherwise, initialization will recalculate the path and replace + any values added to ``module_search_paths``. + +* :c:func:`PyConfig_Read` no longer calculates the initial search path, and will not + fill any values into :c:member:`PyConfig.module_search_paths`. To calculate default + paths and then modify them, finish initialization and use :c:func:`PySys_GetObject` + to retrieve :data:`sys.path` as a Python list object and modify it directly. + Deprecated ---------- diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 840cb061129b7..15f8672adda0d 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -48,7 +48,8 @@ This saves the maintainer the effort of going through the SVN log when researching a change. -This article explains the new features in Python 3.2 as compared to 3.1. It +This article explains the new features in Python 3.2 as compared to 3.1. +Python 3.2 was released on February 20, 2011. It focuses on a few highlights and gives a few examples. For full details, see the `Misc/NEWS `_ @@ -744,7 +745,8 @@ Two methods have been deprecated: * :meth:`xml.etree.ElementTree.getiterator` use ``Element.iter`` instead. For details of the update, see `Introducing ElementTree -`_ on Fredrik Lundh's website. +`_ +on Fredrik Lundh's website. (Contributed by Florent Xicluna and Fredrik Lundh, :issue:`6472`.) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index f4ff143224196..023736134b2ca 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -409,7 +409,7 @@ Some smaller changes made to the core Python language are: evaluating has no elements. (Contributed by Julian Berman in :issue:`18111`.) -* Module objects are now :mod:`weakref`'able. +* Module objects are now :ref:`weakly referenceable `. * Module ``__file__`` attributes (and related values) should now always contain absolute paths by default, with the sole exception of @@ -1113,8 +1113,8 @@ with additional speedups by Antoine Pitrou in :issue:`19219`.) mmap ---- -mmap objects can now be :mod:`weakref`\ ed. (Contributed by Valerie Lambert in -:issue:`4885`.) +mmap objects are now :ref:`weakly referenceable `. +(Contributed by Valerie Lambert in :issue:`4885`.) multiprocessing diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 7c293a501895b..163332401deeb 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -45,6 +45,7 @@ :Editor: Raymond Hettinger This article explains the new features in Python 3.8, compared to 3.7. +Python 3.8 was released on October 14, 2019. For full details, see the :ref:`changelog `. .. testsetup:: @@ -1490,7 +1491,7 @@ Optimizations first introduced in Python 3.4. It offers better performance and smaller size compared to Protocol 3 available since Python 3.0. -* Removed one ``Py_ssize_t`` member from ``PyGC_Head``. All GC tracked +* Removed one :c:type:`Py_ssize_t` member from ``PyGC_Head``. All GC tracked objects (e.g. tuple, list, dict) size is reduced 4 or 8 bytes. (Contributed by Inada Naoki in :issue:`33597`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 6dee55e5a0e55..6deaede4953bd 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -45,7 +45,7 @@ when researching a change. This article explains the new features in Python 3.9, compared to 3.8. -Python 3.9 was released on October 5th, 2020. +Python 3.9 was released on October 5, 2020. For full details, see the :ref:`changelog `. diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 161e2cb30fde3..d276669312ee2 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -111,9 +111,8 @@ static inline PyObject * PyObject_CallMethodOneArg(PyObject *self, PyObject *name, PyObject *arg) { PyObject *args[2] = {self, arg}; - - assert(arg != NULL); size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + assert(arg != NULL); return PyObject_VectorcallMethod(name, args, nargsf, _Py_NULL); } @@ -160,9 +159,8 @@ static inline PyObject * _PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) { PyObject *args[2] = {self, arg}; - - assert(arg != NULL); size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + assert(arg != NULL); return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); } diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index ebbea5ebf1ebc..1add8213e0c09 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -30,16 +30,22 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); // Macros and static inline functions, trading safety for speed -static inline Py_ssize_t PyList_GET_SIZE(PyListObject *op) { - return Py_SIZE(op); +static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { + PyListObject *list = _PyList_CAST(op); + return Py_SIZE(list); } -#define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyList_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op)) +#endif #define PyList_GET_ITEM(op, index) (_PyList_CAST(op)->ob_item[index]) static inline void -PyList_SET_ITEM(PyListObject *op, Py_ssize_t index, PyObject *value) { - op->ob_item[index] = value; +PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { + PyListObject *list = _PyList_CAST(op); + list->ob_item[index] = value; } +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 #define PyList_SET_ITEM(op, index, value) \ - PyList_SET_ITEM(_PyList_CAST(op), index, _PyObject_CAST(value)) + PyList_SET_ITEM(_PyObject_CAST(op), index, _PyObject_CAST(value)) +#endif diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index d5b810e5f4277..3d9c1aff58863 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -19,19 +19,25 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); // Macros and static inline functions, trading safety for speed -static inline Py_ssize_t PyTuple_GET_SIZE(PyTupleObject *op) { - return Py_SIZE(op); +static inline Py_ssize_t PyTuple_GET_SIZE(PyObject *op) { + PyTupleObject *tuple = _PyTuple_CAST(op); + return Py_SIZE(tuple); } -#define PyTuple_GET_SIZE(op) PyTuple_GET_SIZE(_PyTuple_CAST(op)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyTuple_GET_SIZE(op) PyTuple_GET_SIZE(_PyObject_CAST(op)) +#endif #define PyTuple_GET_ITEM(op, index) (_PyTuple_CAST(op)->ob_item[index]) /* Function *only* to be used to fill in brand new tuples */ static inline void -PyTuple_SET_ITEM(PyTupleObject *op, Py_ssize_t index, PyObject *value) { - op->ob_item[index] = value; +PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { + PyTupleObject *tuple = _PyTuple_CAST(op); + tuple->ob_item[index] = value; } +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 #define PyTuple_SET_ITEM(op, index, value) \ - PyTuple_SET_ITEM(_PyTuple_CAST(op), index, _PyObject_CAST(value)) + PyTuple_SET_ITEM(_PyObject_CAST(op), index, _PyObject_CAST(value)) +#endif PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 8e182d0fbf799..84307d1885472 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -326,8 +326,9 @@ static inline void* _PyUnicode_COMPACT_DATA(PyObject *op) { } static inline void* _PyUnicode_NONCOMPACT_DATA(PyObject *op) { + void *data; assert(!PyUnicode_IS_COMPACT(op)); - void *data = _PyUnicodeObject_CAST(op)->data.any; + data = _PyUnicodeObject_CAST(op)->data.any; assert(data != NULL); return data; } @@ -367,7 +368,7 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { kind and data pointers obtained from other function calls. index is the index in the string (starts at 0) and value is the new code point value which should be written to that location. */ -static inline void PyUnicode_WRITE(unsigned int kind, void *data, +static inline void PyUnicode_WRITE(int kind, void *data, Py_ssize_t index, Py_UCS4 value) { if (kind == PyUnicode_1BYTE_KIND) { @@ -384,12 +385,15 @@ static inline void PyUnicode_WRITE(unsigned int kind, void *data, _Py_STATIC_CAST(Py_UCS4*, data)[index] = value; } } +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 #define PyUnicode_WRITE(kind, data, index, value) \ - PyUnicode_WRITE((unsigned int)(kind), (void*)(data), (index), (Py_UCS4)(value)) + PyUnicode_WRITE(_Py_STATIC_CAST(int, kind), _Py_CAST(void*, data), \ + (index), _Py_STATIC_CAST(Py_UCS4, value)) +#endif /* Read a code point from the string's canonical representation. No checks or ready calls are performed. */ -static inline Py_UCS4 PyUnicode_READ(unsigned int kind, +static inline Py_UCS4 PyUnicode_READ(int kind, const void *data, Py_ssize_t index) { if (kind == PyUnicode_1BYTE_KIND) { @@ -401,8 +405,12 @@ static inline Py_UCS4 PyUnicode_READ(unsigned int kind, assert(kind == PyUnicode_4BYTE_KIND); return _Py_STATIC_CAST(const Py_UCS4*, data)[index]; } +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 #define PyUnicode_READ(kind, data, index) \ - PyUnicode_READ((unsigned int)(kind), (const void*)(data), (index)) + PyUnicode_READ(_Py_STATIC_CAST(int, kind), \ + _Py_STATIC_CAST(const void*, data), \ + (index)) +#endif /* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it calls PyUnicode_KIND() and might call it twice. For single reads, use @@ -410,8 +418,9 @@ static inline Py_UCS4 PyUnicode_READ(unsigned int kind, cache kind and use PyUnicode_READ instead. */ static inline Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) { + int kind; assert(PyUnicode_IS_READY(unicode)); - unsigned int kind = PyUnicode_KIND(unicode); + kind = PyUnicode_KIND(unicode); if (kind == PyUnicode_1BYTE_KIND) { return PyUnicode_1BYTE_DATA(unicode)[index]; } @@ -431,12 +440,14 @@ static inline Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) than iterating over the string. */ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) { + int kind; + assert(PyUnicode_IS_READY(op)); if (PyUnicode_IS_ASCII(op)) { return 0x7fU; } - unsigned int kind = PyUnicode_KIND(op); + kind = PyUnicode_KIND(op); if (kind == PyUnicode_1BYTE_KIND) { return 0xffU; } diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index bec69ba90d6b9..26b364f41d4d7 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -37,9 +37,11 @@ PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { + PyWeakReference *ref; + PyObject *obj; assert(PyWeakref_Check(ref_obj)); - PyWeakReference *ref = _Py_CAST(PyWeakReference*, ref_obj); - PyObject *obj = ref->wr_object; + ref = _Py_CAST(PyWeakReference*, ref_obj); + obj = ref->wr_object; // Explanation for the Py_REFCNT() check: when a weakref's target is part // of a long chain of deallocations which triggers the trashcan mechanism, // clearing the weakrefs can be delayed long after the target's refcount @@ -51,4 +53,6 @@ static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { } return Py_None; } -#define PyWeakref_GET_OBJECT(ref) PyWeakref_GET_OBJECT(_PyObject_CAST(ref)) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define PyWeakref_GET_OBJECT(ref) PyWeakref_GET_OBJECT(_PyObject_CAST(ref)) +#endif diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 8dd89c6850794..d969a5e6eb99b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -12,8 +12,14 @@ extern "C" { struct pyruntimestate; struct _ceval_runtime_state; +/* WASI has limited call stack. wasmtime 0.36 can handle sufficient amount of + C stack frames for little more than 750 recursions. */ #ifndef Py_DEFAULT_RECURSION_LIMIT -# define Py_DEFAULT_RECURSION_LIMIT 1000 +# ifdef __wasi__ +# define Py_DEFAULT_RECURSION_LIMIT 750 +# else +# define Py_DEFAULT_RECURSION_LIMIT 1000 +# endif #endif #include "pycore_interp.h" // PyInterpreterState.eval_frame @@ -74,11 +80,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject* const* args, size_t argcount, PyObject *kwnames); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp); -#else extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); -#endif extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); extern void _PyEval_FiniGIL(PyInterpreterState *interp); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 4e1f2ec6c3529..cfa8ae99d1b6d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -201,8 +201,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) + STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_subst__) - STRUCT_FOR_ID(__typing_unpacked__) + STRUCT_FOR_ID(__typing_unpacked_tuple_args__) STRUCT_FOR_ID(__warningregistry__) STRUCT_FOR_ID(__weakref__) STRUCT_FOR_ID(__xor__) diff --git a/Include/internal/pycore_hamt.h b/Include/internal/pycore_hamt.h index 85e35c5afc90c..4d64288bbab49 100644 --- a/Include/internal/pycore_hamt.h +++ b/Include/internal/pycore_hamt.h @@ -5,7 +5,19 @@ # error "this header requires Py_BUILD_CORE define" #endif -#define _Py_HAMT_MAX_TREE_DEPTH 7 + +/* +HAMT tree is shaped by hashes of keys. Every group of 5 bits of a hash denotes +the exact position of the key in one level of the tree. Since we're using +32 bit hashes, we can have at most 7 such levels. Although if there are +two distinct keys with equal hashes, they will have to occupy the same +cell in the 7th level of the tree -- so we'd put them in a "collision" node. +Which brings the total possible tree depth to 8. Read more about the actual +layout of the HAMT tree in `hamt.c`. + +This constant is used to define a datastucture for storing iteration state. +*/ +#define _Py_HAMT_MAX_TREE_DEPTH 8 extern PyTypeObject _PyHamt_Type; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d55627908a28f..bcfcd88d84492 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -51,9 +51,6 @@ struct _ceval_state { /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; struct _pending_calls pending; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state gil; -#endif }; diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index c4bc53c707fda..e2d7c5bee28b9 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -64,18 +64,10 @@ _Py_ThreadCanHandlePendingCalls(void) /* Variable and macro for in-line access to current thread and interpreter state */ -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -PyAPI_FUNC(PyThreadState*) _PyThreadState_GetTSS(void); -#endif - static inline PyThreadState* _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->gilstate.tstate_current); -#endif } /* Get the current Python thread state. @@ -90,11 +82,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) static inline PyThreadState* _PyThreadState_GET(void) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return _PyRuntimeState_GetThreadState(&_PyRuntime); -#endif } PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 18191c3771dfc..ae63ae74afa5f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -23,9 +23,7 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS struct _gil_runtime_state gil; -#endif }; /* GIL state */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c1c5fd562e6b8..57cacb97bcf1a 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -824,8 +824,9 @@ extern "C" { INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ + INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_subst__), \ - INIT_ID(__typing_unpacked__), \ + INIT_ID(__typing_unpacked_tuple_args__), \ INIT_ID(__warningregistry__), \ INIT_ID(__weakref__), \ INIT_ID(__xor__), \ diff --git a/Include/object.h b/Include/object.h index fac8892f65552..f2af428e2bb97 100644 --- a/Include/object.h +++ b/Include/object.h @@ -137,11 +137,12 @@ static inline PyTypeObject* Py_TYPE(PyObject *ob) { #endif // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -static inline Py_ssize_t Py_SIZE(PyVarObject *ob) { - return ob->ob_size; +static inline Py_ssize_t Py_SIZE(PyObject *ob) { + PyVarObject *var_ob = _PyVarObject_CAST(ob); + return var_ob->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SIZE(ob) Py_SIZE(_PyVarObject_CAST(ob)) +# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9747ca4177722..06d20ff72be04 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 11 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.11.0b1" +#define PY_VERSION "3.11.0b3+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pymacro.h b/Include/pymacro.h index 71d6714afd112..b959eeb3f58b3 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -10,6 +10,14 @@ # define static_assert _Static_assert #endif +// static_assert is defined in glibc from version 2.16. Before it requires +// compiler support (gcc >= 4.6) and is called _Static_assert. +#if (defined(__GLIBC__) \ + && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 16)) \ + && !defined(static_assert)) +# define static_assert _Static_assert +#endif + /* Minimum value between x and y */ #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) diff --git a/Include/pyport.h b/Include/pyport.h index 614a2789fb078..a78e290931fff 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -22,12 +22,48 @@ // _Py_CAST(PyObject*, op) can convert a "const PyObject*" to // "PyObject*". // -// The type argument must not be constant. For example, in C++, -// _Py_CAST(const PyObject*, expr) fails with a compiler error. +// The type argument must not be a constant type. #ifdef __cplusplus +#include # define _Py_STATIC_CAST(type, expr) static_cast(expr) -# define _Py_CAST(type, expr) \ - const_cast(reinterpret_cast(expr)) +extern "C++" { + namespace { + template + inline type _Py_CAST_impl(long int ptr) { + return reinterpret_cast(ptr); + } + template + inline type _Py_CAST_impl(int ptr) { + return reinterpret_cast(ptr); + } + template + inline type _Py_CAST_impl(std::nullptr_t) { + return static_cast(nullptr); + } + + template + inline type _Py_CAST_impl(expr_type *expr) { + return reinterpret_cast(expr); + } + + template + inline type _Py_CAST_impl(expr_type const *expr) { + return reinterpret_cast(const_cast(expr)); + } + + template + inline type _Py_CAST_impl(expr_type &expr) { + return static_cast(expr); + } + + template + inline type _Py_CAST_impl(expr_type const &expr) { + return static_cast(const_cast(expr)); + } + } +} +# define _Py_CAST(type, expr) _Py_CAST_impl(expr) + #else # define _Py_STATIC_CAST(type, expr) ((type)(expr)) # define _Py_CAST(type, expr) ((type)(expr)) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 0f647eed99d81..0bfdeaafae274 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -2022,13 +2022,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None, encoding = text_encoding(encoding) if encoding == "locale": - try: - import locale - except ImportError: - # Importing locale may fail if Python is being built - encoding = "utf-8" - else: - encoding = locale.getencoding() + encoding = self._get_locale_encoding() if not isinstance(encoding, str): raise ValueError("invalid encoding: %r" % encoding) @@ -2162,7 +2156,7 @@ def reconfigure(self, *, if not isinstance(encoding, str): raise TypeError("invalid encoding: %r" % encoding) if encoding == "locale": - encoding = locale.getencoding() + encoding = self._get_locale_encoding() if newline is Ellipsis: newline = self._readnl @@ -2267,6 +2261,15 @@ def _get_decoded_chars(self, n=None): self._decoded_chars_used += len(chars) return chars + def _get_locale_encoding(self): + try: + import locale + except ImportError: + # Importing locale may fail if Python is being built + return "utf-8" + else: + return locale.getencoding() + def _rewind_decoded_chars(self, n): """Rewind the _decoded_chars buffer.""" if self._decoded_chars_used < n: diff --git a/Lib/ast.py b/Lib/ast.py index e81e28044bc6e..4e2ae859245f9 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1335,7 +1335,11 @@ def write_item(item): ) def visit_Tuple(self, node): - with self.require_parens(_Precedence.TUPLE, node): + with self.delimit_if( + "(", + ")", + len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE + ): self.items_view(self.traverse, node.elts) unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 9636c6b4d28fa..ddb9daca02693 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -113,7 +113,7 @@ def close(self): def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) - self.close() + self._sock.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): try: diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 6af21f3a15d93..9e0610deed281 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -3,8 +3,6 @@ __all__ = ["TaskGroup"] -import weakref - from . import events from . import exceptions from . import tasks @@ -19,8 +17,7 @@ def __init__(self): self._loop = None self._parent_task = None self._parent_cancel_requested = False - self._tasks = weakref.WeakSet() - self._unfinished_tasks = 0 + self._tasks = set() self._errors = [] self._base_error = None self._on_completed_fut = None @@ -29,8 +26,6 @@ def __repr__(self): info = [''] if self._tasks: info.append(f'tasks={len(self._tasks)}') - if self._unfinished_tasks: - info.append(f'unfinished={self._unfinished_tasks}') if self._errors: info.append(f'errors={len(self._errors)}') if self._aborting: @@ -93,7 +88,7 @@ async def __aexit__(self, et, exc, tb): # can be cancelled multiple times if our parent task # is being cancelled repeatedly (or even once, when # our own cancellation is already in progress) - while self._unfinished_tasks: + while self._tasks: if self._on_completed_fut is None: self._on_completed_fut = self._loop.create_future() @@ -114,7 +109,7 @@ async def __aexit__(self, et, exc, tb): self._on_completed_fut = None - assert self._unfinished_tasks == 0 + assert not self._tasks if self._base_error is not None: raise self._base_error @@ -141,7 +136,7 @@ async def __aexit__(self, et, exc, tb): def create_task(self, coro, *, name=None, context=None): if not self._entered: raise RuntimeError(f"TaskGroup {self!r} has not been entered") - if self._exiting and self._unfinished_tasks == 0: + if self._exiting and not self._tasks: raise RuntimeError(f"TaskGroup {self!r} is finished") if context is None: task = self._loop.create_task(coro) @@ -149,7 +144,6 @@ def create_task(self, coro, *, name=None, context=None): task = self._loop.create_task(coro, context=context) tasks._set_task_name(task, name) task.add_done_callback(self._on_task_done) - self._unfinished_tasks += 1 self._tasks.add(task) return task @@ -169,10 +163,9 @@ def _abort(self): t.cancel() def _on_task_done(self, task): - self._unfinished_tasks -= 1 - assert self._unfinished_tasks >= 0 + self._tasks.discard(task) - if self._on_completed_fut is not None and not self._unfinished_tasks: + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 821034da21adc..7e2f5fa30e826 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -652,6 +652,10 @@ def __init__(self, max_workers=None, mp_context=None, mp_context = mp.get_context() self._mp_context = mp_context + # https://github.com/python/cpython/issues/90622 + self._safe_to_dynamically_spawn_children = ( + self._mp_context.get_start_method(allow_none=False) != "fork") + if initializer is not None and not callable(initializer): raise TypeError("initializer must be a callable") self._initializer = initializer @@ -714,6 +718,8 @@ def __init__(self, max_workers=None, mp_context=None, def _start_executor_manager_thread(self): if self._executor_manager_thread is None: # Start the processes so that their sentinels are known. + if not self._safe_to_dynamically_spawn_children: # ie, using fork. + self._launch_processes() self._executor_manager_thread = _ExecutorManagerThread(self) self._executor_manager_thread.start() _threads_wakeups[self._executor_manager_thread] = \ @@ -726,15 +732,32 @@ def _adjust_process_count(self): process_count = len(self._processes) if process_count < self._max_workers: - p = self._mp_context.Process( - target=_process_worker, - args=(self._call_queue, - self._result_queue, - self._initializer, - self._initargs, - self._max_tasks_per_child)) - p.start() - self._processes[p.pid] = p + # Assertion disabled as this codepath is also used to replace a + # worker that unexpectedly dies, even when using the 'fork' start + # method. That means there is still a potential deadlock bug. If a + # 'fork' mp_context worker dies, we'll be forking a new one when + # we know a thread is running (self._executor_manager_thread). + #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622' + self._spawn_process() + + def _launch_processes(self): + # https://github.com/python/cpython/issues/90622 + assert not self._executor_manager_thread, ( + 'Processes cannot be fork()ed after the thread has started, ' + 'deadlock in the child processes could result.') + for _ in range(len(self._processes), self._max_workers): + self._spawn_process() + + def _spawn_process(self): + p = self._mp_context.Process( + target=_process_worker, + args=(self._call_queue, + self._result_queue, + self._initializer, + self._initargs, + self._max_tasks_per_child)) + p.start() + self._processes[p.pid] = p def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock: @@ -755,7 +778,8 @@ def submit(self, fn, /, *args, **kwargs): # Wake up queue management thread self._executor_manager_thread_wakeup.wakeup() - self._adjust_process_count() + if self._safe_to_dynamically_spawn_children: + self._adjust_process_count() self._start_executor_manager_thread() return f submit.__doc__ = _base.Executor.submit.__doc__ diff --git a/Lib/datetime.py b/Lib/datetime.py index afbb6fed2ecb6..00ded32cc3e3c 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1754,7 +1754,7 @@ def _fromtimestamp(cls, t, utc, tz): y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) ss = min(ss, 59) # clamp out leap seconds if the platform has them result = cls(y, m, d, hh, mm, ss, us, tz) - if tz is None: + if tz is None and not utc: # As of version 2015f max fold in IANA database is # 23 hours at 1969-09-30 13:00:00 in Kwajalein. # Let's probe 24 hours in the past to detect a transition: @@ -1775,7 +1775,7 @@ def _fromtimestamp(cls, t, utc, tz): probe2 = cls(y, m, d, hh, mm, ss, us, tz) if probe2 == result: result._fold = 1 - else: + elif tz is not None: result = tz.fromutc(result) return result diff --git a/Lib/dis.py b/Lib/dis.py index 046013120b000..5a5ee8d848d3a 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -592,7 +592,7 @@ def _unpack_opargs(code): caches = _inline_cache_entries[deop] if deop >= HAVE_ARGUMENT: arg = code[i+1] | extended_arg - extended_arg = (arg << 8) if op == EXTENDED_ARG else 0 + extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0 # The oparg is stored as a signed integer # If the value exceeds its upper limit, it will overflow and wrap # to a negative integer diff --git a/Lib/distutils/tests/test_build.py b/Lib/distutils/tests/test_build.py index 83a9e4f4dd2f4..71b5e164bae14 100644 --- a/Lib/distutils/tests/test_build.py +++ b/Lib/distutils/tests/test_build.py @@ -12,6 +12,7 @@ class BuildTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): + @unittest.skipUnless(sys.executable, "test requires sys.executable") def test_finalize_options(self): pkg_dir, dist = self.create_dist() cmd = build(dist) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 031897bd2734f..4ebeafecef03c 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -41,9 +41,7 @@ def setUp(self): # bpo-30132: On Windows, a .pdb file may be created in the current # working directory. Create a temporary working directory to cleanup # everything at the end of the test. - change_cwd = os_helper.change_cwd(self.tmp_dir) - change_cwd.__enter__() - self.addCleanup(change_cwd.__exit__, None, None, None) + self.enterContext(os_helper.change_cwd(self.tmp_dir)) def tearDown(self): import site diff --git a/Lib/doctest.py b/Lib/doctest.py index ed94d15c0e2da..b2ef2ce63672e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1085,19 +1085,21 @@ def _get_test(self, obj, name, module, globs, source_lines): def _find_lineno(self, obj, source_lines): """ - Return a line number of the given object's docstring. Note: - this method assumes that the object has a docstring. + Return a line number of the given object's docstring. + + Returns `None` if the given object does not have a docstring. """ lineno = None + docstring = getattr(obj, '__doc__', None) # Find the line number for modules. - if inspect.ismodule(obj): + if inspect.ismodule(obj) and docstring is not None: lineno = 0 # Find the line number for classes. # Note: this could be fooled if a class is defined multiple # times in a single file. - if inspect.isclass(obj): + if inspect.isclass(obj) and docstring is not None: if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % @@ -1109,7 +1111,9 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ - if inspect.isfunction(obj): obj = obj.__code__ + if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + # We don't use `docstring` var here, because `obj` can be changed. + obj = obj.__code__ if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 8a8fb8bc42a95..e637e6df06612 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2379,7 +2379,7 @@ def get_section(value): digits += value[0] value = value[1:] if digits[0] == '0' and digits != '0': - section.defects.append(errors.InvalidHeaderError( + section.defects.append(errors.InvalidHeaderDefect( "section number has an invalid leading 0")) section.number = int(digits) section.append(ValueTerminal(digits, 'digits')) diff --git a/Lib/enum.py b/Lib/enum.py index b9811fe9e6787..fe521b1018d48 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -480,8 +480,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k # check for illegal enum names (any others?) invalid_names = set(member_names) & {'mro', ''} if invalid_names: - raise ValueError('invalid enum member name(s) '.format( - ','.join(repr(n) for n in invalid_names))) + raise ValueError('invalid enum member name(s) %s' % ( + ','.join(repr(n) for n in invalid_names) + )) # # adjust the sunders _order_ = classdict.pop('_order_', None) @@ -537,7 +538,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k # # create a default docstring if one has not been provided if enum_class.__doc__ is None: - if not member_names: + if not member_names or not list(enum_class): enum_class.__doc__ = classdict['__doc__'] = _dedent("""\ Create a collection of name/value pairs. @@ -1574,7 +1575,7 @@ def __invert__(self): __rxor__ = __xor__ -class IntFlag(int, ReprEnum, Flag, boundary=EJECT): +class IntFlag(int, ReprEnum, Flag, boundary=KEEP): """ Support for integer-based Flags """ @@ -1643,6 +1644,7 @@ def global_str(self): use enum_name instead of class.enum_name """ if self._name_ is None: + cls_name = self.__class__.__name__ return "%s(%r)" % (cls_name, self._value_) else: return self._name_ diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 0f5a41ac06f3e..d5e296f7748c1 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -102,7 +102,7 @@ def translate(pat): add('\\[') else: stuff = pat[i:j] - if '--' not in stuff: + if '-' not in stuff: stuff = stuff.replace('\\', r'\\') else: chunks = [] @@ -114,7 +114,16 @@ def translate(pat): chunks.append(pat[i:k]) i = k+1 k = k+3 - chunks.append(pat[i:j]) + chunk = pat[i:j] + if chunk: + chunks.append(chunk) + else: + chunks[-1] += '-' + # Remove empty ranges -- invalid in RE. + for k in range(len(chunks)-1, 0, -1): + if chunks[k-1][-1] > chunks[k][0]: + chunks[k-1] = chunks[k-1][:-1] + chunks[k][1:] + del chunks[k] # Escape backslashes and hyphens for set difference (--). # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') @@ -122,11 +131,18 @@ def translate(pat): # Escape set operations (&&, ~~ and ||). stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 - if stuff[0] == '!': - stuff = '^' + stuff[1:] - elif stuff[0] in ('^', '['): - stuff = '\\' + stuff - add(f'[{stuff}]') + if not stuff: + # Empty range: never match. + add('(?!)') + elif stuff == '!': + # Negated empty range: match any character. + add('.') + else: + if stuff[0] == '!': + stuff = '^' + stuff[1:] + elif stuff[0] in ('^', '['): + stuff = '\\' + stuff + add(f'[{stuff}]') else: add(re.escape(c)) assert i == n diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index 4a41d8433d548..2e10d7cd36760 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -83,12 +83,12 @@ def test_paren_corner(self): """ Test corner cases in flash_paren_event and paren_closed_event. - These cases force conditional expression and alternate paths. + Force execution of conditional expressions and alternate paths. """ text = self.text pm = self.get_parenmatch() - text.insert('insert', '# this is a commen)') + text.insert('insert', '# Comment.)') pm.paren_closed_event('event') text.insert('insert', '\ndef') diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index e2dddbec577c0..8e7773bcae13e 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -63,7 +63,7 @@ def test_load_grammar_from_pickle(self): @unittest.skipIf(sys.executable is None, 'sys.executable required') @unittest.skipIf( - sys.platform == 'emscripten', 'requires working subprocess' + sys.platform in {'emscripten', 'wasi'}, 'requires working subprocess' ) def test_load_grammar_from_subprocess(self): tmpdir = tempfile.mkdtemp() diff --git a/Lib/locale.py b/Lib/locale.py index 25eb75ac65a32..7a7694e1bfb71 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -633,7 +633,17 @@ def resetlocale(category=LC_ALL): getdefaultlocale(). category defaults to LC_ALL. """ - _setlocale(category, _build_localename(getdefaultlocale())) + import warnings + warnings.warn( + 'Use locale.setlocale(locale.LC_ALL, "") instead', + DeprecationWarning, stacklevel=2 + ) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=DeprecationWarning) + loc = getdefaultlocale() + + _setlocale(category, _build_localename(loc)) try: diff --git a/Lib/mailcap.py b/Lib/mailcap.py index 856b6a55475f3..7278ea7051fcc 100644 --- a/Lib/mailcap.py +++ b/Lib/mailcap.py @@ -2,6 +2,7 @@ import os import warnings +import re __all__ = ["getcaps","findmatch"] @@ -19,6 +20,11 @@ def lineno_sort_key(entry): else: return 1, 0 +_find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search + +class UnsafeMailcapInput(Warning): + """Warning raised when refusing unsafe input""" + # Part 1: top-level interface. @@ -171,15 +177,22 @@ def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]): entry to use. """ + if _find_unsafe(filename): + msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,) + warnings.warn(msg, UnsafeMailcapInput) + return None, None entries = lookup(caps, MIMEtype, key) # XXX This code should somehow check for the needsterminal flag. for e in entries: if 'test' in e: test = subst(e['test'], filename, plist) + if test is None: + continue if test and os.system(test) != 0: continue command = subst(e[key], MIMEtype, filename, plist) - return command, e + if command is not None: + return command, e return None, None def lookup(caps, MIMEtype, key=None): @@ -212,6 +225,10 @@ def subst(field, MIMEtype, filename, plist=[]): elif c == 's': res = res + filename elif c == 't': + if _find_unsafe(MIMEtype): + msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,) + warnings.warn(msg, UnsafeMailcapInput) + return None res = res + MIMEtype elif c == '{': start = i @@ -219,7 +236,12 @@ def subst(field, MIMEtype, filename, plist=[]): i = i+1 name = field[start:i] i = i+1 - res = res + findparam(name, plist) + param = findparam(name, plist) + if _find_unsafe(param): + msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name) + warnings.warn(msg, UnsafeMailcapInput) + return None + res = res + param # XXX To do: # %n == number of parts if type is multipart/* # %F == list of alternating type and filename for parts diff --git a/Lib/os.py b/Lib/os.py index 67662ca7ad858..648188e0f1349 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -974,15 +974,14 @@ def spawnlpe(mode, file, *args): # command in a shell can't be supported. if sys.platform != 'vxworks': # Supply os.popen() - def popen(cmd, mode="r", buffering=-1, encoding=None): + def popen(cmd, mode="r", buffering=-1): if not isinstance(cmd, str): raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) if mode not in ("r", "w"): raise ValueError("invalid mode %r" % mode) if buffering == 0 or buffering is None: raise ValueError("popen() does not support unbuffered streams") - import subprocess, io - encoding = io.text_encoding(encoding) + import subprocess if mode == "r": proc = subprocess.Popen(cmd, shell=True, text=True, diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 1f098fe6bd5f3..9d40d88ef5f2e 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -443,6 +443,8 @@ def __getitem__(self, idx): if idx >= len(self) or idx < -len(self): raise IndexError(idx) + if idx < 0: + idx += len(self) return self._pathcls._from_parsed_parts(self._drv, self._root, self._parts[:-idx - 1]) @@ -960,7 +962,7 @@ def rglob(self, pattern): drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") - if pattern[-1] in (self._flavour.sep, self._flavour.altsep): + if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): pattern_parts.append('') selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour) for p in selector.select_from(self): diff --git a/Lib/platform.py b/Lib/platform.py index 3f3f25a2c92d3..c272c407c7776 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -186,6 +186,10 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): executable = sys.executable + if not executable: + # sys.executable is not set. + return lib, version + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index f3ceaadfad645..eaae3630dfe28 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Fri May 6 23:53:34 2022 +# Autogenerated by Sphinx on Wed Jun 1 14:07:03 2022 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -4861,7 +4861,10 @@ 'is\n' 'applied to separating the commands; the input is split at the ' 'first\n' - '";;" pair, even if it is in the middle of a quoted string.\n' + '";;" pair, even if it is in the middle of a quoted string. A\n' + 'workaround for strings with double semicolons is to use ' + 'implicit\n' + 'string concatenation "\';\'\';\'" or "";"";"".\n' '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' @@ -8254,7 +8257,7 @@ '| "x(arguments...)", "x.attribute" | ' 'attribute reference |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "await" "x" | ' + '| "await x" | ' 'Await expression |\n' '+-------------------------------------------------+---------------------------------------+\n' '| "**" | ' @@ -8290,7 +8293,7 @@ '| ">=", "!=", "==" | ' 'tests and identity tests |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "not" "x" | ' + '| "not x" | ' 'Boolean NOT |\n' '+-------------------------------------------------+---------------------------------------+\n' '| "and" | ' diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index c9b511422f1e5..d58c2117ef3e1 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -129,7 +129,7 @@ # public symbols __all__ = [ "match", "fullmatch", "search", "sub", "subn", "split", - "findall", "finditer", "compile", "purge", "escape", + "findall", "finditer", "compile", "purge", "template", "escape", "error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE", "UNICODE", "NOFLAG", "RegexFlag", @@ -148,6 +148,8 @@ class RegexFlag: MULTILINE = M = _compiler.SRE_FLAG_MULTILINE # make anchors look for newline DOTALL = S = _compiler.SRE_FLAG_DOTALL # make dot match newline VERBOSE = X = _compiler.SRE_FLAG_VERBOSE # ignore whitespace and comments + # sre extensions (experimental, don't rely on these) + TEMPLATE = T = _compiler.SRE_FLAG_TEMPLATE # unknown purpose, deprecated DEBUG = _compiler.SRE_FLAG_DEBUG # dump pattern after compilation __str__ = object.__str__ _numeric_repr_ = hex @@ -229,6 +231,18 @@ def purge(): _cache.clear() _compile_repl.cache_clear() +def template(pattern, flags=0): + "Compile a template pattern, returning a Pattern object, deprecated" + import warnings + warnings.warn("The re.template() function is deprecated " + "as it is an undocumented function " + "without an obvious purpose. " + "Use re.compile() instead.", + DeprecationWarning) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) # warn just once + return _compile(pattern, flags|T) + # SPECIAL_CHARS # closing ')', '}' and ']' # '-' (a range in character set) @@ -270,6 +284,13 @@ def _compile(pattern, flags): return pattern if not _compiler.isstring(pattern): raise TypeError("first argument must be string or compiled pattern") + if flags & T: + import warnings + warnings.warn("The re.TEMPLATE/re.T flag is deprecated " + "as it is an undocumented flag " + "without an obvious purpose. " + "Don't use it.", + DeprecationWarning) p = _compiler.compile(pattern, flags) if not (flags & DEBUG): if len(_cache) >= _MAXCACHE: diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 63d82025505b7..4b5322338cbd5 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -108,6 +108,8 @@ def _compile(data, pattern, flags): else: emit(ANY) elif op in REPEATING_CODES: + if flags & SRE_FLAG_TEMPLATE: + raise error("internal: unsupported template operator %r" % (op,)) if _simple(av[2]): emit(REPEATING_CODES[op][2]) skip = _len(code); emit(0) diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py index 71204d903b322..1cc85c631f22b 100644 --- a/Lib/re/_constants.py +++ b/Lib/re/_constants.py @@ -204,6 +204,7 @@ def _makecodes(*names): } # flags +SRE_FLAG_TEMPLATE = 1 # template mode (unknown purpose, deprecated) SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_MULTILINE = 8 # treat target as multiline string diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index a393c508d86e5..f747a0396b1f3 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -61,11 +61,12 @@ "x": SRE_FLAG_VERBOSE, # extensions "a": SRE_FLAG_ASCII, + "t": SRE_FLAG_TEMPLATE, "u": SRE_FLAG_UNICODE, } TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE -GLOBAL_FLAGS = SRE_FLAG_DEBUG +GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE class State: # keeps track of state for parsing diff --git a/Lib/socket.py b/Lib/socket.py old mode 100755 new mode 100644 diff --git a/Lib/statistics.py b/Lib/statistics.py index 54f4e13265189..2d66b0522f19d 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -611,7 +611,7 @@ def median_high(data): return data[n // 2] -def median_grouped(data, interval=1): +def median_grouped(data, interval=1.0): """Estimates the median for numeric data binned around the midpoints of consecutive, fixed-width intervals. @@ -650,35 +650,34 @@ def median_grouped(data, interval=1): by exact multiples of *interval*. This is essential for getting a correct result. The function does not check this precondition. + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. + """ data = sorted(data) n = len(data) - if n == 0: + if not n: raise StatisticsError("no median for empty data") - elif n == 1: - return data[0] # Find the value at the midpoint. Remember this corresponds to the # midpoint of the class interval. x = data[n // 2] - # Generate a clear error message for non-numeric data - for obj in (x, interval): - if isinstance(obj, (str, bytes)): - raise TypeError(f'expected a number but got {obj!r}') - # Using O(log n) bisection, find where all the x values occur in the data. # All x will lie within data[i:j]. i = bisect_left(data, x) j = bisect_right(data, x, lo=i) + # Coerce to floats, raising a TypeError if not possible + try: + interval = float(interval) + x = float(x) + except ValueError: + raise TypeError(f'Value cannot be converted to a float') + # Interpolate the median using the formula found at: # https://www.cuemath.com/data/median-of-grouped-data/ - try: - L = x - interval / 2 # The lower limit of the median interval. - except TypeError: - # Coerce mixed types to float. - L = float(x) - float(interval) / 2 + L = x - interval / 2.0 # Lower limit of the median interval cf = i # Cumulative frequency of the preceding interval f = j - i # Number of elements in the median internal return L + interval * (n / 2 - cf) / f diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 6e61cc2e5e7b0..e10b01047ebef 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -74,6 +74,9 @@ else: _mswindows = True +# wasm32-emscripten and wasm32-wasi do not support processes +_can_fork_exec = sys.platform not in {"emscripten", "wasi"} + if _mswindows: import _winapi from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, @@ -97,13 +100,10 @@ "CREATE_NO_WINDOW", "DETACHED_PROCESS", "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) else: - if sys.platform in {"emscripten", "wasi"}: - def _fork_exec(*args, **kwargs): - raise OSError( - errno.ENOTSUP, f"{sys.platform} does not support processes." - ) - else: + if _can_fork_exec: from _posixsubprocess import fork_exec as _fork_exec + else: + _fork_exec = None import select import selectors @@ -801,6 +801,11 @@ def __init__(self, args, bufsize=-1, executable=None, encoding=None, errors=None, text=None, umask=-1, pipesize=-1, process_group=None): """Create new Popen instance.""" + if not _can_fork_exec: + raise OSError( + errno.ENOTSUP, f"{sys.platform} does not support processes." + ) + _cleanup() # Held while anything is calling waitpid before returncode has been # updated to prevent clobbering returncode if wait() or poll() are diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index e7b7106e17851..7ff641b37bf18 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -236,10 +236,11 @@ def test_path(self): module_search_paths=['a', 'b', 'c']) self.assertEqual(sys.path, ['a', 'b', 'c']) - # Leave sys.path unchanged if module_search_paths_set=0 + # sys.path is reset if module_search_paths_set=0 self.set_config(module_search_paths_set=0, module_search_paths=['new_path']) - self.assertEqual(sys.path, ['a', 'b', 'c']) + self.assertNotEqual(sys.path, ['a', 'b', 'c']) + self.assertNotEqual(sys.path, ['new_path']) def test_argv(self): self.set_config(parse_argv=0, diff --git a/Lib/test/_testcppext.cpp b/Lib/test/_testcppext.cpp index dc40f0ee9eb1c..5e3d76b7b2072 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/_testcppext.cpp @@ -23,6 +23,26 @@ _testcppext_add(PyObject *Py_UNUSED(module), PyObject *args) } +// Class to test operator casting an object to PyObject* +class StrongRef +{ +public: + StrongRef(PyObject *obj) : m_obj(obj) { + Py_INCREF(this->m_obj); + } + + ~StrongRef() { + Py_DECREF(this->m_obj); + } + + // Cast to PyObject*: get a borrowed reference + inline operator PyObject*() const { return this->m_obj; } + +private: + PyObject *m_obj; // Strong reference +}; + + static PyObject * test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { @@ -30,6 +50,8 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) if (obj == nullptr) { return nullptr; } + Py_ssize_t refcnt = Py_REFCNT(obj); + assert(refcnt >= 1); // gh-92138: For backward compatibility, functions of Python C API accepts // "const PyObject*". Check that using it does not emit C++ compiler @@ -38,21 +60,66 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_INCREF(const_obj); Py_DECREF(const_obj); PyTypeObject *type = Py_TYPE(const_obj); - assert(Py_REFCNT(const_obj) >= 1); - + assert(Py_REFCNT(const_obj) == refcnt); assert(type == &PyTuple_Type); assert(PyTuple_GET_SIZE(const_obj) == 2); PyObject *one = PyTuple_GET_ITEM(const_obj, 0); assert(PyLong_AsLong(one) == 1); + // gh-92898: StrongRef doesn't inherit from PyObject but has an operator to + // cast to PyObject*. + StrongRef strong_ref(obj); + assert(Py_TYPE(strong_ref) == &PyTuple_Type); + assert(Py_REFCNT(strong_ref) == (refcnt + 1)); + Py_INCREF(strong_ref); + Py_DECREF(strong_ref); + + // gh-93442: Pass 0 as NULL for PyObject* + Py_XINCREF(0); + Py_XDECREF(0); + // ensure that nullptr works too + Py_XINCREF(nullptr); + Py_XDECREF(nullptr); + Py_DECREF(obj); Py_RETURN_NONE; } +static PyObject * +test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyObject *str = PyUnicode_FromString("abc"); + if (str == nullptr) { + return nullptr; + } + + assert(PyUnicode_Check(str)); + assert(PyUnicode_GET_LENGTH(str) == 3); + + // gh-92800: test PyUnicode_READ() + const void* data = PyUnicode_DATA(str); + assert(data != nullptr); + int kind = PyUnicode_KIND(str); + assert(kind == PyUnicode_1BYTE_KIND); + assert(PyUnicode_READ(kind, data, 0) == 'a'); + + // gh-92800: test PyUnicode_READ() casts + const void* const_data = PyUnicode_DATA(str); + unsigned int ukind = static_cast(kind); + assert(PyUnicode_READ(ukind, const_data, 2) == 'c'); + + assert(PyUnicode_READ_CHAR(str, 1) == 'b'); + + Py_DECREF(str); + Py_RETURN_NONE; +} + + static PyMethodDef _testcppext_methods[] = { {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc}, {"test_api_casts", test_api_casts, METH_NOARGS, nullptr}, + {"test_unicode", test_unicode, METH_NOARGS, nullptr}, {nullptr, nullptr, 0, nullptr} /* sentinel */ }; diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0495362b3f369..8e4bcc7c9efdc 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2515,45 +2515,101 @@ def test_microsecond_rounding(self): self.assertEqual(t.microsecond, 7812) def test_timestamp_limits(self): - # minimum timestamp - min_dt = self.theclass.min.replace(tzinfo=timezone.utc) + with self.subTest("minimum UTC"): + min_dt = self.theclass.min.replace(tzinfo=timezone.utc) + min_ts = min_dt.timestamp() + + # This test assumes that datetime.min == 0000-01-01T00:00:00.00 + # If that assumption changes, this value can change as well + self.assertEqual(min_ts, -62135596800) + + with self.subTest("maximum UTC"): + # Zero out microseconds to avoid rounding issues + max_dt = self.theclass.max.replace(tzinfo=timezone.utc, + microsecond=0) + max_ts = max_dt.timestamp() + + # This test assumes that datetime.max == 9999-12-31T23:59:59.999999 + # If that assumption changes, this value can change as well + self.assertEqual(max_ts, 253402300799.0) + + def test_fromtimestamp_limits(self): + try: + self.theclass.fromtimestamp(-2**32 - 1) + except (OSError, OverflowError): + self.skipTest("Test not valid on this platform") + + # XXX: Replace these with datetime.{min,max}.timestamp() when we solve + # the issue with gh-91012 + min_dt = self.theclass.min + timedelta(days=1) min_ts = min_dt.timestamp() + + max_dt = self.theclass.max.replace(microsecond=0) + max_ts = ((self.theclass.max - timedelta(hours=23)).timestamp() + + timedelta(hours=22, minutes=59, seconds=59).total_seconds()) + + for (test_name, ts, expected) in [ + ("minimum", min_ts, min_dt), + ("maximum", max_ts, max_dt), + ]: + with self.subTest(test_name, ts=ts, expected=expected): + actual = self.theclass.fromtimestamp(ts) + + self.assertEqual(actual, expected) + + # Test error conditions + test_cases = [ + ("Too small by a little", min_ts - timedelta(days=1, hours=12).total_seconds()), + ("Too small by a lot", min_ts - timedelta(days=400).total_seconds()), + ("Too big by a little", max_ts + timedelta(days=1).total_seconds()), + ("Too big by a lot", max_ts + timedelta(days=400).total_seconds()), + ] + + for test_name, ts in test_cases: + with self.subTest(test_name, ts=ts): + with self.assertRaises((ValueError, OverflowError)): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.fromtimestamp(ts) + + def test_utcfromtimestamp_limits(self): try: - # date 0001-01-01 00:00:00+00:00: timestamp=-62135596800 - self.assertEqual(self.theclass.fromtimestamp(min_ts, tz=timezone.utc), - min_dt) - except (OverflowError, OSError) as exc: - # the date 0001-01-01 doesn't fit into 32-bit time_t, - # or platform doesn't support such very old date - self.skipTest(str(exc)) - - # maximum timestamp: set seconds to zero to avoid rounding issues - max_dt = self.theclass.max.replace(tzinfo=timezone.utc, - second=0, microsecond=0) + self.theclass.utcfromtimestamp(-2**32 - 1) + except (OSError, OverflowError): + self.skipTest("Test not valid on this platform") + + min_dt = self.theclass.min.replace(tzinfo=timezone.utc) + min_ts = min_dt.timestamp() + + max_dt = self.theclass.max.replace(microsecond=0, tzinfo=timezone.utc) max_ts = max_dt.timestamp() - # date 9999-12-31 23:59:00+00:00: timestamp 253402300740 - self.assertEqual(self.theclass.fromtimestamp(max_ts, tz=timezone.utc), - max_dt) - - # number of seconds greater than 1 year: make sure that the new date - # is not valid in datetime.datetime limits - delta = 3600 * 24 * 400 - - # too small - ts = min_ts - delta - # converting a Python int to C time_t can raise a OverflowError, - # especially on 32-bit platforms. - with self.assertRaises((ValueError, OverflowError)): - self.theclass.fromtimestamp(ts) - with self.assertRaises((ValueError, OverflowError)): - self.theclass.utcfromtimestamp(ts) - - # too big - ts = max_dt.timestamp() + delta - with self.assertRaises((ValueError, OverflowError)): - self.theclass.fromtimestamp(ts) - with self.assertRaises((ValueError, OverflowError)): - self.theclass.utcfromtimestamp(ts) + + for (test_name, ts, expected) in [ + ("minimum", min_ts, min_dt.replace(tzinfo=None)), + ("maximum", max_ts, max_dt.replace(tzinfo=None)), + ]: + with self.subTest(test_name, ts=ts, expected=expected): + try: + actual = self.theclass.utcfromtimestamp(ts) + except (OSError, OverflowError) as exc: + self.skipTest(str(exc)) + + self.assertEqual(actual, expected) + + # Test error conditions + test_cases = [ + ("Too small by a little", min_ts - 1), + ("Too small by a lot", min_ts - timedelta(days=400).total_seconds()), + ("Too big by a little", max_ts + 1), + ("Too big by a lot", max_ts + timedelta(days=400).total_seconds()), + ] + + for test_name, ts in test_cases: + with self.subTest(test_name, ts=ts): + with self.assertRaises((ValueError, OverflowError)): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.utcfromtimestamp(ts) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/doctest_lineno.py b/Lib/test/doctest_lineno.py new file mode 100644 index 0000000000000..be198513a1502 --- /dev/null +++ b/Lib/test/doctest_lineno.py @@ -0,0 +1,50 @@ +# This module is used in `test_doctest`. +# It must not have a docstring. + +def func_with_docstring(): + """Some unrelated info.""" + + +def func_without_docstring(): + pass + + +def func_with_doctest(): + """ + This function really contains a test case. + + >>> func_with_doctest.__name__ + 'func_with_doctest' + """ + return 3 + + +class ClassWithDocstring: + """Some unrelated class information.""" + + +class ClassWithoutDocstring: + pass + + +class ClassWithDoctest: + """This class really has a test case in it. + + >>> ClassWithDoctest.__name__ + 'ClassWithDoctest' + """ + + +class MethodWrapper: + def method_with_docstring(self): + """Method with a docstring.""" + + def method_without_docstring(self): + pass + + def method_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.method_with_doctest.__name__ + 'method_with_doctest' + """ diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 28549a645b0f5..2339e0049ef6e 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -10,6 +10,9 @@ import warnings +MS_WINDOWS = (sys.platform == 'win32') + + def normalize_text(text): if text is None: return None @@ -127,13 +130,21 @@ def collect_sys(info_add): encoding = '%s/%s' % (encoding, errors) info_add('sys.%s.encoding' % name, encoding) - # Were we compiled --with-pydebug or with #define Py_DEBUG? + # Were we compiled --with-pydebug? Py_DEBUG = hasattr(sys, 'gettotalrefcount') if Py_DEBUG: text = 'Yes (sys.gettotalrefcount() present)' else: text = 'No (sys.gettotalrefcount() missing)' - info_add('Py_DEBUG', text) + info_add('build.Py_DEBUG', text) + + # Were we compiled --with-trace-refs? + Py_TRACE_REFS = hasattr(sys, 'getobjects') + if Py_TRACE_REFS: + text = 'Yes (sys.getobjects() present)' + else: + text = 'No (sys.getobjects() missing)' + info_add('build.Py_TRACE_REFS', text) def collect_platform(info_add): @@ -455,6 +466,11 @@ def collect_datetime(info_add): def collect_sysconfig(info_add): + # On Windows, sysconfig is not reliable to get macros used + # to build Python + if MS_WINDOWS: + return + import sysconfig for name in ( @@ -488,6 +504,28 @@ def collect_sysconfig(info_add): value = normalize_text(value) info_add('sysconfig[%s]' % name, value) + PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') + NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS) + if NDEBUG: + text = 'ignore assertions (macro defined)' + else: + text= 'build assertions (macro not defined)' + info_add('build.NDEBUG',text) + + for name in ( + 'WITH_DOC_STRINGS', + 'WITH_DTRACE', + 'WITH_FREELISTS', + 'WITH_PYMALLOC', + 'WITH_VALGRIND', + ): + value = sysconfig.get_config_var(name) + if value: + text = 'Yes' + else: + text = 'No' + info_add(f'build.{name}', text) + def collect_ssl(info_add): import os @@ -543,10 +581,19 @@ def format_attr(attr, value): def collect_socket(info_add): - import socket + try: + import socket + except ImportError: + return - hostname = socket.gethostname() - info_add('socket.hostname', hostname) + try: + hostname = socket.gethostname() + except OSError: + # WASI SDK 15.0 does not have gethostname(2). + if sys.platform != "wasi": + raise + else: + info_add('socket.hostname', hostname) def collect_sqlite(info_add): @@ -596,7 +643,6 @@ def collect_testcapi(info_add): return call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname') - copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC') def collect_resource(info_add): @@ -638,6 +684,13 @@ def collect_test_support(info_add): call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available') call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized') + info_add('test_support.check_sanitizer(address=True)', + support.check_sanitizer(address=True)) + info_add('test_support.check_sanitizer(memory=True)', + support.check_sanitizer(memory=True)) + info_add('test_support.check_sanitizer(ub=True)', + support.check_sanitizer(ub=True)) + def collect_cc(info_add): import subprocess diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3b2f33979db9a..35c4efb01af27 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -199,6 +199,11 @@ def get_original_stdout(): def _force_run(path, func, *args): try: return func(*args) + except FileNotFoundError as err: + # chmod() won't fix a missing file. + if verbose >= 2: + print('%s: %s' % (err.__class__.__name__, err)) + raise except OSError as err: if verbose >= 2: print('%s: %s' % (err.__class__.__name__, err)) @@ -299,6 +304,8 @@ def requires(resource, msg=None): if msg is None: msg = "Use of the %r resource not enabled" % resource raise ResourceDenied(msg) + if resource in {"network", "urlfetch"} and not has_socket_support: + raise ResourceDenied("No socket support") if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) @@ -521,7 +528,7 @@ def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") -# Emscripten's socket emulation has limitation. WASI doesn't have sockets yet. +# Emscripten's socket emulation and WASI sockets have limitations. has_socket_support = not is_emscripten and not is_wasi def requires_working_socket(*, module=False): diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index eee37ef0d5a71..5d787f16b69dd 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -50,7 +50,7 @@ % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None # macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten'}: +elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) @@ -171,9 +171,13 @@ def can_symlink(): global _can_symlink if _can_symlink is not None: return _can_symlink - symlink_path = TESTFN + "can_symlink" + # WASI / wasmtime prevents symlinks with absolute paths, see man + # openat2(2) RESOLVE_BENEATH. Almost all symlink tests use absolute + # paths. Skip symlink tests on WASI for now. + src = os.path.abspath(TESTFN) + symlink_path = src + "can_symlink" try: - os.symlink(TESTFN, symlink_path) + os.symlink(src, symlink_path) can = True except (OSError, NotImplementedError, AttributeError): can = False diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 754af181ec922..42b2a93398cbf 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -11,6 +11,9 @@ HOSTv4 = "127.0.0.1" HOSTv6 = "::1" +# WASI SDK 15.0 does not provide gethostname, stub raises OSError ENOTSUP. +has_gethostname = not support.is_wasi + def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): """Returns an unused port that should be suitable for binding. This is diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index b3bc54cd55104..0947464bb8c04 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -109,7 +109,8 @@ def numeric_tester(self, calc_type, calc_value, data_type, used_locale): @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" ) def test_lc_numeric_nl_langinfo(self): # Test nl_langinfo against known values @@ -128,7 +129,8 @@ def test_lc_numeric_nl_langinfo(self): self.skipTest('no suitable locales') @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten, bpo-46390" ) def test_lc_numeric_localeconv(self): # Test localeconv against known values diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 907ae27d529b5..4a14cb352138e 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -19,8 +19,7 @@ def setUp(self): self.maxDiff = None self.prog_name = 'bogus_program_xxxx' self.temp_path_dir = os.path.abspath(os.getcwd()) - self.env = os_helper.EnvironmentVarGuard() - self.addCleanup(self.env.__exit__) + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 8509deb93f1e2..273db45c00f7a 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -41,9 +41,8 @@ def setUp(self): # The tests assume that line wrapping occurs at 80 columns, but this # behaviour can be overridden by setting the COLUMNS environment # variable. To ensure that this width is used, set COLUMNS to 80. - env = os_helper.EnvironmentVarGuard() + env = self.enterContext(os_helper.EnvironmentVarGuard()) env['COLUMNS'] = '80' - self.addCleanup(env.__exit__) class TempDirMixin(object): @@ -3428,9 +3427,8 @@ class TestShortColumns(HelpTestCase): but we don't want any exceptions thrown in such cases. Only ugly representation. ''' def setUp(self): - env = os_helper.EnvironmentVarGuard() + env = self.enterContext(os_helper.EnvironmentVarGuard()) env.set("COLUMNS", '15') - self.addCleanup(env.__exit__) parser_signature = TestHelpBiggerOptionals.parser_signature argument_signatures = TestHelpBiggerOptionals.argument_signatures diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 03d9b310f161b..33df22cb35a9e 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -335,6 +335,41 @@ def test_ast_validation(self): for snippet in snippets_to_validate: tree = ast.parse(snippet) compile(tree, '', 'exec') + + def test_invalid_position_information(self): + invalid_linenos = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] + + for lineno, end_lineno in invalid_linenos: + with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].lineno = lineno + tree.body[0].end_lineno = end_lineno + with self.assertRaises(ValueError): + compile(tree, '', 'exec') + + invalid_col_offsets = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] + for col_offset, end_col_offset in invalid_col_offsets: + with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].col_offset = col_offset + tree.body[0].end_col_offset = end_col_offset + with self.assertRaises(ValueError): + compile(tree, '', 'exec') + + def test_compilation_of_ast_nodes_with_default_end_position_values(self): + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) + + # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. + compile(tree, "", "exec") def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 9ba6ba04b32b8..5e3c1573c9c58 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -82,7 +82,7 @@ def tearDown(self): def tcp_server(self, server_prog, *, family=socket.AF_INET, addr=None, - timeout=5, + timeout=support.SHORT_TIMEOUT, backlog=1, max_clients=10): @@ -113,7 +113,7 @@ def tcp_server(self, server_prog, *, def tcp_client(self, client_prog, family=socket.AF_INET, - timeout=10): + timeout=support.SHORT_TIMEOUT): sock = socket.socket(family, socket.SOCK_STREAM) @@ -238,7 +238,7 @@ def prog(sock): async def start_server(): extras = {} - extras = dict(ssl_handshake_timeout=40.0) + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) srv = await asyncio.start_server( handle_client, @@ -303,7 +303,7 @@ def server(sock): async def client(addr): extras = {} - extras = dict(ssl_handshake_timeout=40.0) + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) reader, writer = await asyncio.open_connection( *addr, @@ -428,7 +428,7 @@ async def client(addr): *addr, ssl=client_sslctx, server_hostname='', - ssl_handshake_timeout=1.0) + ssl_handshake_timeout=support.SHORT_TIMEOUT) writer.close() await self.wait_closed(writer) @@ -590,7 +590,7 @@ def client(): extras = {} if server_ssl: - extras = dict(ssl_handshake_timeout=10.0) + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) f = loop.create_task( loop.connect_accepted_socket( @@ -718,7 +718,8 @@ async def client(addr): with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: self.loop.run_until_complete( - asyncio.wait_for(client(srv.addr), timeout=10)) + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) def test_create_connection_memory_leak(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE @@ -776,7 +777,8 @@ async def client(addr): with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: self.loop.run_until_complete( - asyncio.wait_for(client(srv.addr), timeout=10)) + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) # No garbage is left for SSL client from loop.create_connection, even # if user stores the SSLTransport in corresponding protocol instance @@ -936,7 +938,8 @@ async def client(addr): with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: self.loop.run_until_complete( - asyncio.wait_for(client(srv.addr), timeout=10)) + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) def test_start_tls_server_1(self): HELLO_MSG = b'1' * self.PAYLOAD_SIZE @@ -1186,7 +1189,7 @@ def server(sock): async def client(addr): extras = {} - extras = dict(ssl_handshake_timeout=10.0) + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) reader, writer = await asyncio.open_connection( *addr, diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index e0feef7c65360..7ac063cfc78d3 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -82,6 +82,7 @@ def f(): self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_callback_on_subinterpreter_teardown(self): # This tests if a callback is called on # subinterpreter teardown. diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 20f8b9d7c0aa8..ba108221ebf45 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -257,6 +257,12 @@ def test_insort(self): target ) + def test_insort_keynotNone(self): + x = [] + y = {"a": 2, "b": 1} + for f in (self.module.insort_left, self.module.insort_right): + self.assertRaises(TypeError, f, x, y, key = "b") + class TestBisectPython(TestBisect, unittest.TestCase): module = py_bisect diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index ba7a7e20d7dcd..efa9459a58629 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -393,7 +393,10 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - @unittest.skipIf(support.is_emscripten, "socket.accept is broken") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "socket.accept is broken" + ) def test_compile_top_level_await(self): """Test whether code some top level await can be compiled. diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6d75895589328..904ae9bc47ecf 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -788,6 +788,7 @@ def test_pendingcalls_non_threaded(self): class SubinterpreterTest(unittest.TestCase): + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_subinterps(self): import builtins r, w = os.pipe() @@ -803,6 +804,7 @@ def test_subinterps(self): self.assertNotEqual(pickle.load(f), id(sys.modules)) self.assertNotEqual(pickle.load(f), id(builtins)) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_subinterps_recent_language_features(self): r, w = os.pipe() code = """if 1: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index bb433dc1e73a4..d783af65839ad 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -558,8 +558,9 @@ def test_non_ascii(self): # Mac OS X denies the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but # Python cannot a undecodable bytes argument to a subprocess. + # WASI does not permit invalid UTF-8 names. if (os_helper.TESTFN_UNDECODABLE - and sys.platform not in ('win32', 'darwin')): + and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): name = os.fsdecode(os_helper.TESTFN_UNDECODABLE) elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 5a9c618786f4e..c32c27f33b447 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -109,7 +109,9 @@ def __getitem__(self, key): self.assertEqual(d['z'], 12) def test_extended_arg(self): - longexpr = 'x = x or ' + '-x' * 2500 + # default: 1000 * 2.5 = 2500 repetitions + repeat = int(sys.getrecursionlimit() * 2.5) + longexpr = 'x = x or ' + '-x' * repeat g = {} code = ''' def f(x): diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 1a9fc973c3aa1..73c83c9bf1efe 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -434,6 +434,9 @@ class CompileallTestsWithoutSourceEpoch(CompileallTestsBase, pass +# WASI does not have a temp directory and uses cwd instead. The cwd contains +# non-ASCII chars, so _walk_dir() fails to encode self.directory. +@unittest.skipIf(support.is_wasi, "tempdir is not encodable on WASI") class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors to stdout.""" diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 4363e90b8bbab..6f3b4609232bb 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -497,10 +497,16 @@ def acquire_lock(lock): lock.acquire() mp_context = self.get_context() + if mp_context.get_start_method(allow_none=False) == "fork": + # fork pre-spawns, not on demand. + expected_num_processes = self.worker_count + else: + expected_num_processes = 3 + sem = mp_context.Semaphore(0) for _ in range(3): self.executor.submit(acquire_lock, sem) - self.assertEqual(len(self.executor._processes), 3) + self.assertEqual(len(self.executor._processes), expected_num_processes) for _ in range(3): sem.release() processes = self.executor._processes @@ -1021,6 +1027,8 @@ def test_saturation(self): def test_idle_process_reuse_one(self): executor = self.executor assert executor._max_workers >= 4 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") executor.submit(mul, 21, 2).result() executor.submit(mul, 6, 7).result() executor.submit(mul, 3, 14).result() @@ -1029,6 +1037,8 @@ def test_idle_process_reuse_one(self): def test_idle_process_reuse_multiple(self): executor = self.executor assert executor._max_workers <= 5 + if self.get_context().get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") executor.submit(mul, 12, 7).result() executor.submit(mul, 33, 25) executor.submit(mul, 25, 26).result() diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 3132cea668cb1..b1aece4f5c9c4 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -535,6 +535,41 @@ def test_hamt_collision_1(self): self.assertEqual(len(h4), 2) self.assertEqual(len(h5), 3) + def test_hamt_collision_3(self): + # Test that iteration works with the deepest tree possible. + # https://github.com/python/cpython/issues/93065 + + C = HashKey(0b10000000_00000000_00000000_00000000, 'C') + D = HashKey(0b10000000_00000000_00000000_00000000, 'D') + + E = HashKey(0b00000000_00000000_00000000_00000000, 'E') + + h = hamt() + h = h.set(C, 'C') + h = h.set(D, 'D') + h = h.set(E, 'E') + + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=2 count=1 bitmap=0b1): + # NULL: + # BitmapNode(size=4 count=2 bitmap=0b101): + # : 'E' + # NULL: + # CollisionNode(size=4 id=0x107a24520): + # : 'C' + # : 'D' + + self.assertEqual({k.name for k in h.keys()}, {'C', 'D', 'E'}) + def test_hamt_stress(self): COLLECTION_SIZE = 7000 TEST_ITERS_EVERY = 647 diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 76bd81c7d02cc..b64673d2c31e0 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -15,15 +15,12 @@ def _async_test(func): @functools.wraps(func) def wrapper(*args, **kwargs): coro = func(*args, **kwargs) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - return loop.run_until_complete(coro) - finally: - loop.close() - asyncio.set_event_loop_policy(None) + asyncio.run(coro) return wrapper +def tearDownModule(): + asyncio.set_event_loop_policy(None) + class TestAbstractAsyncContextManager(unittest.TestCase): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 77944e678c750..dba5ceffaf1c0 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2209,7 +2209,8 @@ async def f(): @unittest.skipIf( - support.is_emscripten, "asyncio does not work under Emscripten yet." + support.is_emscripten or support.is_wasi, + "asyncio does not work under Emscripten/WASI yet." ) class CoroAsyncIOCompatTest(unittest.TestCase): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 202b99829f213..edcac8ee2b7c5 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -633,6 +633,22 @@ def loop_test(): loop_test.__code__.co_firstlineno + 2, loop_test.__code__.co_firstlineno + 1,) +def extended_arg_quick(): + *_, _ = ... + +dis_extended_arg_quick_code = """\ +%3d 0 RESUME 0 + +%3d 2 LOAD_CONST 1 (Ellipsis) + 4 EXTENDED_ARG_QUICK 1 + 6 UNPACK_EX 256 + 8 STORE_FAST 0 (_) + 10 STORE_FAST 0 (_) + 12 LOAD_CONST 0 (None) + 14 RETURN_VALUE +"""% (extended_arg_quick.__code__.co_firstlineno, + extended_arg_quick.__code__.co_firstlineno + 1,) + QUICKENING_WARMUP_DELAY = 8 class DisTestBase(unittest.TestCase): @@ -1011,6 +1027,11 @@ def test_loop_quicken(self): got = self.get_disassembly(loop_test, adaptive=True) self.do_disassembly_compare(got, dis_loop_test_quickened_code, True) + @cpython_only + def test_extended_arg_quick(self): + got = self.get_disassembly(extended_arg_quick) + self.do_disassembly_compare(got, dis_extended_arg_quick_code, True) + def get_cached_values(self, quickened, adaptive): def f(): l = [] @@ -1490,7 +1511,7 @@ def _prepare_test_cases(): Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=368, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=378, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=21, argval=426, argrepr='to 426', offset=382, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=4, argval=392, argrepr='to 392', offset=382, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=384, starts_line=22, is_jump_target=True, positions=None), Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=386, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=388, starts_line=None, is_jump_target=False, positions=None), @@ -1502,24 +1523,16 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=420, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=422, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=424, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=426, starts_line=23, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=428, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=426, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=428, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=440, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='PRECALL', opcode=166, arg=1, argval=1, argrepr='', offset=442, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=446, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=456, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=458, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=460, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=462, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=464, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=476, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PRECALL', opcode=166, arg=1, argval=1, argrepr='', offset=478, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=482, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=492, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=494, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=496, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=498, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=500, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=458, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=460, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=462, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=464, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3e7f3782d89f4..a4aab6cf4db3b 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -25,6 +25,7 @@ # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. +# There are also related tests in `test_doctest2` module. ###################################################################### ## Sample Objects (used by test cases) @@ -460,7 +461,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -642,6 +643,26 @@ def basics(): r""" 1 SampleClass.double 1 SampleClass.get +When used with `exclude_empty=False` we are also interested in line numbers +of doctests that are empty. +It used to be broken for quite some time until `bpo-28249`. + + >>> from test import doctest_lineno + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(doctest_lineno) + >>> for t in tests: + ... print('%5s %s' % (t.lineno, t.name)) + None test.doctest_lineno + 22 test.doctest_lineno.ClassWithDocstring + 30 test.doctest_lineno.ClassWithDoctest + None test.doctest_lineno.ClassWithoutDocstring + None test.doctest_lineno.MethodWrapper + 39 test.doctest_lineno.MethodWrapper.method_with_docstring + 45 test.doctest_lineno.MethodWrapper.method_with_doctest + None test.doctest_lineno.MethodWrapper.method_without_docstring + 4 test.doctest_lineno.func_with_docstring + 12 test.doctest_lineno.func_with_doctest + None test.doctest_lineno.func_without_docstring + Turning off Recursion ~~~~~~~~~~~~~~~~~~~~~ DocTestFinder can be told not to look for tests in contained objects diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 8a436ad123b80..4b971deacc1a5 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -6,9 +6,14 @@ import types import unittest +from test import support from test.support import findfile +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + def abspath(filename): return os.path.abspath(findfile(filename, subdir="dtracedata")) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 69f883a3673f2..2b1e2b864feda 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -19,24 +19,25 @@ import email.policy from email.charset import Charset -from email.header import Header, decode_header, make_header -from email.parser import Parser, HeaderParser from email.generator import Generator, DecodedGenerator, BytesGenerator +from email.header import Header, decode_header, make_header +from email.headerregistry import HeaderRegistry from email.message import Message from email.mime.application import MIMEApplication from email.mime.audio import MIMEAudio -from email.mime.text import MIMEText -from email.mime.image import MIMEImage from email.mime.base import MIMEBase +from email.mime.image import MIMEImage from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart -from email import utils -from email import errors +from email.mime.text import MIMEText +from email.parser import Parser, HeaderParser +from email import base64mime from email import encoders +from email import errors from email import iterators -from email import base64mime from email import quoprimime +from email import utils from test.support import threading_helper from test.support.os_helper import unlink @@ -5541,7 +5542,12 @@ def test_long_headers_flatten(self): result = fp.getvalue() self._signed_parts_eq(original, result) - +class TestHeaderRegistry(TestEmailBase): + # See issue gh-93010. + def test_HeaderRegistry(self): + reg = HeaderRegistry() + a = reg('Content-Disposition', 'attachment; 0*00="foo"') + self.assertIsInstance(a.defects[0], errors.InvalidHeaderDefect) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 169ae5cb0a06e..5ba6e3a43fdc6 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -343,19 +343,39 @@ def test_finalize_structseq(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) - @support.skip_if_pgo_task def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 - code = """if 1: - from importlib._bootstrap import _handle_fromlist - import dis - for name in dis.opmap: - # quicken this frozen code object. - _handle_fromlist(dis, [name], lambda *args: None) - """ + + # Do these imports outside of the code string to avoid using + # importlib too much from within the code string, so that + # _handle_fromlist doesn't get quickened until we intend it to. + from dis import _all_opmap + resume = _all_opmap["RESUME"] + resume_quick = _all_opmap["RESUME_QUICK"] + from test.test_dis import QUICKENING_WARMUP_DELAY + + code = textwrap.dedent(f"""\ + import importlib._bootstrap + func = importlib._bootstrap._handle_fromlist + code = func.__code__ + + # Assert initially unquickened. + # Use sets to account for byte order. + if set(code._co_code_adaptive[:2]) != set([{resume}, 0]): + raise AssertionError() + + for i in range({QUICKENING_WARMUP_DELAY}): + func(importlib._bootstrap, ["x"], lambda *args: None) + + # Assert quickening worked + if set(code._co_code_adaptive[:2]) != set([{resume_quick}, 0]): + raise AssertionError() + + print("Tests passed") + """) run = self.run_embedded_interpreter - for i in range(50): - out, err = run("test_repeated_init_exec", code, timeout=60) + out, err = run("test_repeated_init_exec", code) + self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) def test_ucnhash_capi_reset(self): # bpo-47182: unicodeobject.c:ucnhash_capi was not reset on shutdown. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index f9e09027228b4..56cebfea3f1ba 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -189,6 +189,12 @@ class HeadlightsC(IntFlag, boundary=enum.CONFORM): FOG_C = auto() +@enum.global_enum +class NoName(Flag): + ONE = 1 + TWO = 2 + + # tests class _EnumTests: @@ -614,6 +620,7 @@ class _PlainOutputTests: def test_str(self): TE = self.MainEnum if self.is_flag: + self.assertEqual(str(TE(0)), "MainEnum(0)") self.assertEqual(str(TE.dupe), "MainEnum.dupe") self.assertEqual(str(self.dupe2), "MainEnum.first|third") else: @@ -3238,6 +3245,10 @@ def test_global_repr_conform1(self): '%(m)s.OFF_C' % {'m': SHORT_MODULE}, ) + def test_global_enum_str(self): + self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') + self.assertEqual(str(NoName(0)), 'NoName(0)') + def test_format(self): Perm = self.Perm self.assertEqual(format(Perm.R, ''), '4') @@ -3338,7 +3349,10 @@ def test_invert(self): self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) def test_boundary(self): - self.assertIs(enum.IntFlag._boundary_, EJECT) + self.assertIs(enum.IntFlag._boundary_, KEEP) + class Simple(IntFlag, boundary=KEEP): + SINGLE = 1 + # class Iron(IntFlag, boundary=STRICT): ONE = 1 TWO = 2 @@ -3357,7 +3371,6 @@ class Space(IntFlag, boundary=EJECT): EIGHT = 8 self.assertIs(Space._boundary_, EJECT) # - # class Bizarre(IntFlag, boundary=KEEP): b = 3 c = 4 @@ -3374,6 +3387,12 @@ class Bizarre(IntFlag, boundary=KEEP): self.assertEqual(list(Bizarre), [Bizarre.c]) self.assertIs(Bizarre(3), Bizarre.b) self.assertIs(Bizarre(6), Bizarre.d) + # + simple = Simple.SINGLE | Iron.TWO + self.assertEqual(simple, 3) + self.assertIsInstance(simple, Simple) + self.assertEqual(repr(simple), ': 3>') + self.assertEqual(str(simple), '3') def test_iter(self): Color = self.Color diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index e4984d3cd559e..c26cdc028cc89 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -9,7 +9,9 @@ from weakref import proxy from functools import wraps -from test.support import cpython_only, swap_attr, gc_collect, is_emscripten +from test.support import ( + cpython_only, swap_attr, gc_collect, is_emscripten, is_wasi +) from test.support.os_helper import (TESTFN, TESTFN_UNICODE, make_bad_fd) from test.support.warnings_helper import check_warnings from collections import UserList @@ -65,6 +67,7 @@ def testAttributes(self): self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops') + @unittest.skipIf(is_wasi, "WASI does not expose st_blksize.") def testBlksize(self): # test private _blksize attribute blksize = io.DEFAULT_BUFFER_SIZE diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index ca695d6f3f019..10ed496d4e2f3 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -2,6 +2,7 @@ import unittest import os +import string import warnings from fnmatch import fnmatch, fnmatchcase, translate, filter @@ -91,6 +92,119 @@ def test_sep(self): check('usr/bin', 'usr\\bin', normsep) check('usr\\bin', 'usr\\bin') + def test_char_set(self): + ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') + check = self.check_match + tescases = string.ascii_lowercase + string.digits + string.punctuation + for c in tescases: + check(c, '[az]', c in 'az') + check(c, '[!az]', c not in 'az') + # Case insensitive. + for c in tescases: + check(c, '[AZ]', (c in 'az') and ignorecase) + check(c, '[!AZ]', (c not in 'az') or not ignorecase) + for c in string.ascii_uppercase: + check(c, '[az]', (c in 'AZ') and ignorecase) + check(c, '[!az]', (c not in 'AZ') or not ignorecase) + # Repeated same character. + for c in tescases: + check(c, '[aa]', c == 'a') + # Special cases. + for c in tescases: + check(c, '[^az]', c in '^az') + check(c, '[[az]', c in '[az') + check(c, r'[!]]', c != ']') + check('[', '[') + check('[]', '[]') + check('[!', '[!') + check('[!]', '[!]') + + def test_range(self): + ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + tescases = string.ascii_lowercase + string.digits + string.punctuation + for c in tescases: + check(c, '[b-d]', c in 'bcd') + check(c, '[!b-d]', c not in 'bcd') + check(c, '[b-dx-z]', c in 'bcdxyz') + check(c, '[!b-dx-z]', c not in 'bcdxyz') + # Case insensitive. + for c in tescases: + check(c, '[B-D]', (c in 'bcd') and ignorecase) + check(c, '[!B-D]', (c not in 'bcd') or not ignorecase) + for c in string.ascii_uppercase: + check(c, '[b-d]', (c in 'BCD') and ignorecase) + check(c, '[!b-d]', (c not in 'BCD') or not ignorecase) + # Upper bound == lower bound. + for c in tescases: + check(c, '[b-b]', c == 'b') + # Special cases. + for c in tescases: + check(c, '[!-#]', c not in '-#') + check(c, '[!--.]', c not in '-.') + check(c, '[^-`]', c in '^_`') + if not (normsep and c == '/'): + check(c, '[[-^]', c in r'[\]^') + check(c, r'[\-^]', c in r'\]^') + check(c, '[b-]', c in '-b') + check(c, '[!b-]', c not in '-b') + check(c, '[-b]', c in '-b') + check(c, '[!-b]', c not in '-b') + check(c, '[-]', c in '-') + check(c, '[!-]', c not in '-') + # Upper bound is less that lower bound: error in RE. + for c in tescases: + check(c, '[d-b]', False) + check(c, '[!d-b]', True) + check(c, '[d-bx-z]', c in 'xyz') + check(c, '[!d-bx-z]', c not in 'xyz') + check(c, '[d-b^-`]', c in '^_`') + if not (normsep and c == '/'): + check(c, '[d-b[-^]', c in r'[\]^') + + def test_sep_in_char_set(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + check('/', r'[/]') + check('\\', r'[\]') + check('/', r'[\]', normsep) + check('\\', r'[/]', normsep) + check('[/]', r'[/]', False) + check(r'[\\]', r'[/]', False) + check('\\', r'[\t]') + check('/', r'[\t]', normsep) + check('t', r'[\t]') + check('\t', r'[\t]', False) + + def test_sep_in_range(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + check('a/b', 'a[.-0]b', not normsep) + check('a\\b', 'a[.-0]b', False) + check('a\\b', 'a[Z-^]b', not normsep) + check('a/b', 'a[Z-^]b', False) + + check('a/b', 'a[/-0]b', not normsep) + check(r'a\b', 'a[/-0]b', False) + check('a[/-0]b', 'a[/-0]b', False) + check(r'a[\-0]b', 'a[/-0]b', False) + + check('a/b', 'a[.-/]b') + check(r'a\b', 'a[.-/]b', normsep) + check('a[.-/]b', 'a[.-/]b', False) + check(r'a[.-\]b', 'a[.-/]b', False) + + check(r'a\b', r'a[\-^]b') + check('a/b', r'a[\-^]b', normsep) + check(r'a[\-^]b', r'a[\-^]b', False) + check('a[/-^]b', r'a[\-^]b', False) + + check(r'a\b', r'a[Z-\]b', not normsep) + check('a/b', r'a[Z-\]b', False) + check(r'a[Z-\]b', r'a[Z-\]b', False) + check('a[Z-/]b', r'a[Z-\]b', False) + def test_warnings(self): with warnings.catch_warnings(): warnings.simplefilter('error', Warning) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 0c3372f033551..93aa229709a0a 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1073,6 +1073,7 @@ def test_mismatched_braces(self): "f'{'", "f'x{<'", # See bpo-46762. "f'x{>'", + "f'{i='", # See gh-93418. ]) # But these are just normal strings. diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 5fba74ec864f1..6959c2ae3c80e 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -487,5 +487,25 @@ def test_del_iter(self): del iter_x +class TypeIterationTests(unittest.TestCase): + _UNITERABLE_TYPES = (list, tuple) + + def test_cannot_iterate(self): + for test_type in self._UNITERABLE_TYPES: + with self.subTest(type=test_type): + expected_error_regex = "object is not iterable" + with self.assertRaisesRegex(TypeError, expected_error_regex): + iter(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + list(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + for _ in test_type: + pass + + def test_is_not_instance_of_iterable(self): + for type_to_test in self._UNITERABLE_TYPES: + self.assertNotIsInstance(type_to_test, Iterable) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 2741adc139bcf..489044f8090d3 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -484,7 +484,7 @@ def test_nonascii_abspath(self): # invalid UTF-8 name. Windows allows creating a directory with an # arbitrary bytes name, but fails to enter this directory # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten')): + and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index 9261276ebb972..64b9ce01e05ea 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -11,14 +11,10 @@ class GetoptTests(unittest.TestCase): def setUp(self): - self.env = EnvironmentVarGuard() + self.env = self.enterContext(EnvironmentVarGuard()) if "POSIXLY_CORRECT" in self.env: del self.env["POSIXLY_CORRECT"] - def tearDown(self): - self.env.__exit__() - del self.env - def assertError(self, *args, **kwargs): self.assertRaises(getopt.GetoptError, *args, **kwargs) diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 467652a41f0cd..1608d1b18e98f 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -117,6 +117,7 @@ class GettextBaseTest(unittest.TestCase): def setUp(self): + self.addCleanup(os_helper.rmtree, os.path.split(LOCALEDIR)[0]) if not os.path.isdir(LOCALEDIR): os.makedirs(LOCALEDIR) with open(MOFILE, 'wb') as fp: @@ -129,14 +130,10 @@ def setUp(self): fp.write(base64.decodebytes(UMO_DATA)) with open(MMOFILE, 'wb') as fp: fp.write(base64.decodebytes(MMO_DATA)) - self.env = os_helper.EnvironmentVarGuard() + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) self.env['LANGUAGE'] = 'xx' gettext._translations.clear() - def tearDown(self): - self.env.__exit__() - del self.env - os_helper.rmtree(os.path.split(LOCALEDIR)[0]) GNU_MO_DATA_ISSUE_17898 = b'''\ 3hIElQAAAAABAAAAHAAAACQAAAAAAAAAAAAAAAAAAAAsAAAAggAAAC0AAAAAUGx1cmFsLUZvcm1z diff --git a/Lib/test/test_global.py b/Lib/test/test_global.py index d0bde3fd040e6..f5b38c25ea072 100644 --- a/Lib/test/test_global.py +++ b/Lib/test/test_global.py @@ -9,14 +9,9 @@ class GlobalTests(unittest.TestCase): def setUp(self): - self._warnings_manager = check_warnings() - self._warnings_manager.__enter__() + self.enterContext(check_warnings()) warnings.filterwarnings("error", module="") - def tearDown(self): - self._warnings_manager.__exit__(None, None, None) - - def test1(self): prog_text_1 = """\ def wrong1(): @@ -54,9 +49,7 @@ def test4(self): def setUpModule(): - cm = warnings.catch_warnings() - cm.__enter__() - unittest.addModuleCleanup(cm.__exit__, None, None, None) + unittest.enterModuleContext(warnings.catch_warnings()) warnings.filterwarnings("error", module="") diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index ff13edea2d1e4..ed26fa16d6ea8 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -939,6 +939,7 @@ def test_with_statement_logout(self): @threading_helper.reap_threads @cpython_only + @unittest.skipUnless(__debug__, "Won't work if __debug__ is False") def test_dump_ur(self): # See: http://bugs.python.org/issue26543 untagged_resp_dict = {'READ-WRITE': [b'']} diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index be88677dc697e..35c260bd634fc 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,7 +20,8 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten) + STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, + is_wasi) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -535,7 +536,10 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") - @unittest.skipIf(is_emscripten, "Emscripten's umask is a stub.") + @unittest.skipIf( + is_emscripten or is_wasi, + "Emscripten's/WASI's umask is a stub." + ) def test_creation_mode(self): mask = 0o022 with temp_umask(mask), _ready_to_import() as (name, path): diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 6a23e9d50f6ff..bed9d56dca84e 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -157,21 +157,12 @@ def test_dir_removal_handling(self): def test_no_read_directory(self): # Issue #16730 tempdir = tempfile.TemporaryDirectory() + self.enterContext(tempdir) + # Since we muck with the permissions, we want to set them back to + # their original values to make sure the directory can be properly + # cleaned up. original_mode = os.stat(tempdir.name).st_mode - def cleanup(tempdir): - """Cleanup function for the temporary directory. - - Since we muck with the permissions, we want to set them back to - their original values to make sure the directory can be properly - cleaned up. - - """ - os.chmod(tempdir.name, original_mode) - # If this is not explicitly called then the __del__ method is used, - # but since already mucking around might as well explicitly clean - # up. - tempdir.__exit__(None, None, None) - self.addCleanup(cleanup, tempdir) + self.addCleanup(os.chmod, tempdir.name, original_mode) os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR) finder = self.get_finder(tempdir.name) found = self._find(finder, 'doesnotexist') diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 2ea41b7a4c5c3..cd08498545e80 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -65,12 +65,7 @@ def setUp(self): self.resolved_paths = [ os.path.join(self.root, path) for path in self.paths ] - self.ctx = namespace_tree_context(path=self.resolved_paths) - self.ctx.__enter__() - - def tearDown(self): - # TODO: will we ever want to pass exc_info to __exit__? - self.ctx.__exit__(None, None, None) + self.enterContext(namespace_tree_context(path=self.resolved_paths)) class SingleNamespacePackage(NamespacePackageTest): diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index cc1d804f35f91..9aeeb5e686e93 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -272,4 +272,4 @@ def setUpModule(): if __name__ == "__main__": - unittets.main() + unittest.main() diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index fe0259ab609c7..ae1842704d37d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -842,7 +842,10 @@ def test_nested_class_definition_inside_function(self): self.assertSourceEqual(mod2.cls213, 218, 222) self.assertSourceEqual(mod2.cls213().func219(), 220, 221) - @unittest.skipIf(support.is_emscripten, "socket.accept is broken") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "socket.accept is broken" + ) def test_nested_class_definition_inside_async_function(self): import asyncio self.addCleanup(asyncio.set_event_loop_policy, None) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5528c461e58ae..daccbae5b4a1d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -429,6 +429,7 @@ def test_invalid_operations(self): @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_optional_abilities(self): # Test for OSError when optional APIs are not supported # The purpose of this test is to try fileno(), reading, writing and @@ -3569,6 +3570,10 @@ def seekable(self): return True F.tell = lambda x: 0 t = self.TextIOWrapper(F(), encoding='utf-8') + def test_reconfigure_locale(self): + wrapper = io.TextIOWrapper(io.BytesIO(b"test")) + wrapper.reconfigure(encoding="locale") + def test_reconfigure_encoding_read(self): # latin1 -> utf8 # (latin1 can decode utf-8 encoded string) @@ -3970,6 +3975,7 @@ def test_removed_u_mode(self): @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_open_pipe_with_append(self): # bpo-27805: Ignore ESPIPE from lseek() in open(). r, w = os.pipe() @@ -4108,6 +4114,7 @@ def cleanup_fds(): with warnings_helper.check_no_resource_warning(self): open(r, *args, closefd=False, **kwargs) + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): self._check_warn_on_dealloc_fd("rb", buffering=0) self._check_warn_on_dealloc_fd("rb") @@ -4147,6 +4154,7 @@ def test_nonblock_pipe_write_smallbuf(self): @unittest.skipUnless(hasattr(os, 'set_blocking'), 'os.set_blocking() required for this test') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def _test_nonblock_pipe_write(self, bufsize): sent = [] received = [] @@ -4458,14 +4466,17 @@ def _read(): raise @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_unbuffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_buffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb") @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 8f6bec1620053..3c11c59baef6e 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -156,6 +156,8 @@ def test_seekable(self): def skip_no_disk_space(path, required): def decorator(fun): def wrapper(*args, **kwargs): + if not hasattr(shutil, "disk_usage"): + raise unittest.SkipTest("requires shutil.disk_usage") if shutil.disk_usage(os.path.realpath(path)).free < required: hsize = int(required / 1024 / 1024) raise unittest.SkipTest( diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 2e10c55e339d5..e9f6fcb7d7f24 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -244,6 +244,17 @@ def script(self, content, encoding="utf-8"): finally: file.unlink() + @contextlib.contextmanager + def fake_venv(self): + venv = Path.cwd() / "Scripts" + venv.mkdir(exist_ok=True, parents=True) + venv_exe = (venv / Path(sys.executable).name) + venv_exe.touch() + try: + yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} + finally: + shutil.rmtree(venv) + class TestLauncher(unittest.TestCase, RunPyMixin): @classmethod @@ -451,12 +462,8 @@ def test_py_default_in_list(self): self.assertEqual("PythonTestSuite/3.100", default) def test_virtualenv_in_list(self): - venv = Path.cwd() / "Scripts" - venv.mkdir(exist_ok=True, parents=True) - venv_exe = (venv / Path(sys.executable).name) - venv_exe.touch() - try: - data = self.run_py(["-0p"], env={"VIRTUAL_ENV": str(venv.parent)}) + with self.fake_venv() as (venv_exe, env): + data = self.run_py(["-0p"], env=env) for line in data["stdout"].splitlines(): m = re.match(r"\s*\*\s+(.+)$", line) if m: @@ -465,7 +472,7 @@ def test_virtualenv_in_list(self): else: self.fail("did not find active venv path") - data = self.run_py(["-0"], env={"VIRTUAL_ENV": str(venv.parent)}) + data = self.run_py(["-0"], env=env) for line in data["stdout"].splitlines(): m = re.match(r"\s*\*\s+(.+)$", line) if m: @@ -473,8 +480,17 @@ def test_virtualenv_in_list(self): break else: self.fail("did not find active venv entry") - finally: - shutil.rmtree(venv) + + def test_virtualenv_with_env(self): + with self.fake_venv() as (venv_exe, env): + data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + # Compare stdout, because stderr goes via ascii + self.assertEqual(data1["stdout"].strip(), str(venv_exe)) + self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") + # Ensure passing the argument doesn't trigger the same behaviour + self.assertNotEqual(data2["stdout"].strip(), str(venv_exe)) + self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") def test_py_shebang(self): with self.py_ini(TEST_PY_COMMANDS): diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 5cb6edc52d777..bc8a7a35fbf2d 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,5 +1,5 @@ from decimal import Decimal -from test.support import verbose, is_android, is_emscripten +from test.support import verbose, is_android, is_emscripten, is_wasi from test.support.warnings_helper import check_warnings import unittest import locale @@ -373,13 +373,19 @@ def setUp(self): @unittest.skipIf(sys.platform.startswith('aix'), 'bpo-29972: broken test on AIX') - @unittest.skipIf(is_emscripten, "musl libc issue on Emscripten, bpo-46390") + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @unittest.skipIf(sys.platform.startswith('aix'), 'bpo-29972: broken test on AIX') - @unittest.skipIf(is_emscripten, "musl libc issue on Emscripten, bpo-46390") + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 5d4ddedd059fc..fd562322a7637 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5280,6 +5280,7 @@ def test_emit_after_closing_in_write_mode(self): self.assertEqual(fp.read().strip(), '1') class RotatingFileHandlerTest(BaseFileTest): + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): # If maxbytes is zero rollover never occurs rh = logging.handlers.RotatingFileHandler( @@ -5387,6 +5388,7 @@ def rotator(source, dest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): # See bpo-45401. Should only ever rollover regular files fh = logging.handlers.TimedRotatingFileHandler( @@ -5650,9 +5652,7 @@ def test__all__(self): # why the test does this, but in any case we save the current locale # first and restore it at the end. def setUpModule(): - cm = support.run_with_locale('LC_ALL', '') - cm.__enter__() - unittest.addModuleCleanup(cm.__exit__, None, None, None) + unittest.enterModuleContext(support.run_with_locale('LC_ALL', '')) if __name__ == "__main__": diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 20c460e300cc9..07c2764dfd1b2 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -10,12 +10,17 @@ import tempfile from test import support from test.support import os_helper +from test.support import socket_helper import unittest import textwrap import mailbox import glob +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + class TestBase: all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage, diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py index 97a8fac6e074a..8185f4a780661 100644 --- a/Lib/test/test_mailcap.py +++ b/Lib/test/test_mailcap.py @@ -128,7 +128,8 @@ def test_subst(self): (["", "audio/*", "foo.txt"], ""), (["echo foo", "audio/*", "foo.txt"], "echo foo"), (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"), - (["echo %t", "audio/*", "foo.txt"], "echo audio/*"), + (["echo %t", "audio/*", "foo.txt"], None), + (["echo %t", "audio/wav", "foo.txt"], "echo audio/wav"), (["echo \\%t", "audio/*", "foo.txt"], "echo %t"), (["echo foo", "audio/*", "foo.txt", plist], "echo foo"), (["echo %{total}", "audio/*", "foo.txt", plist], "echo 3") @@ -212,7 +213,10 @@ def test_findmatch(self): ('"An audio fragment"', audio_basic_entry)), ([c, "audio/*"], {"filename": fname}, - ("/usr/local/bin/showaudio audio/*", audio_entry)), + (None, None)), + ([c, "audio/wav"], + {"filename": fname}, + ("/usr/local/bin/showaudio audio/wav", audio_entry)), ([c, "message/external-body"], {"plist": plist}, ("showexternal /dev/null default john python.org /tmp foo bar", message_entry)) @@ -221,6 +225,10 @@ def test_findmatch(self): @unittest.skipUnless(os.name == "posix", "Requires 'test' command on system") @unittest.skipIf(sys.platform == "vxworks", "'test' command is not supported on VxWorks") + @unittest.skipUnless( + test.support.has_subprocess_support, + "'test' command needs process support." + ) def test_test(self): # findmatch() will automatically check any "test" conditions and skip # the entry if the check fails. diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 3cca1e8ff1ac1..05a23e5e8a3ce 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,6 +1,11 @@ import netrc, os, unittest, sys, textwrap from test.support import os_helper, run_unittest +try: + import pwd +except ImportError: + pwd = None + temp_filename = os_helper.TESTFN class NetrcTestCase(unittest.TestCase): @@ -266,6 +271,7 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): @unittest.skipUnless(os.name == 'posix', 'POSIX only test') + @unittest.skipIf(pwd is None, 'security check requires pwd module') def test_security(self): # This test is incomplete since we are normally not run as root and # therefore can't test the file ownership being wrong. diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index 9812c05519351..31a02f86abb00 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -1593,8 +1593,7 @@ def setUp(self): self.background.start() self.addCleanup(self.background.join) - self.nntp = NNTP(socket_helper.HOST, port, usenetrc=False).__enter__() - self.addCleanup(self.nntp.__exit__, None, None, None) + self.nntp = self.enterContext(NNTP(socket_helper.HOST, port, usenetrc=False)) def run_server(self, sock): # Could be generalized to handle more commands in separate methods diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 36ad587760d70..f304c5e01e5fa 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -11,7 +11,6 @@ import fractions import itertools import locale -import mmap import os import pickle import select @@ -59,6 +58,10 @@ except ImportError: INT_MAX = PY_SSIZE_T_MAX = sys.maxsize +try: + import mmap +except ImportError: + mmap = None from test.support.script_helper import assert_python_ok from test.support import unix_shell @@ -183,6 +186,9 @@ def test_access(self): @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) + @unittest.skipIf( + support.is_wasi, "WASI does not support dup." + ) def test_closerange(self): first = os.open(os_helper.TESTFN, os.O_CREAT|os.O_RDWR) # We must allocate two consecutive file descriptors, otherwise @@ -1585,7 +1591,10 @@ def test_makedir(self): 'dir5', 'dir6') os.makedirs(path) - @unittest.skipIf(support.is_emscripten, "Emscripten's umask is a stub.") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "Emscripten's/WASI's umask is a stub." + ) def test_mode(self): with os_helper.temp_umask(0o002): base = os_helper.TESTFN @@ -1762,6 +1771,7 @@ def test_remove_nothing(self): self.assertTrue(os.path.exists(os_helper.TESTFN)) +@unittest.skipIf(support.is_wasi, "WASI has no /dev/null") class DevNullTests(unittest.TestCase): def test_devnull(self): with open(os.devnull, 'wb', 0) as f: @@ -2108,6 +2118,7 @@ def test_chmod(self): self.assertRaises(OSError, os.chmod, os_helper.TESTFN, 0) +@unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] @@ -2167,7 +2178,8 @@ def test_fchown(self): @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( - support.is_emscripten, "musl libc issue on Emscripten, bpo-46390" + support.is_emscripten or support.is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" ) def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") @@ -2460,6 +2472,7 @@ def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) + @unittest.skipIf(mmap is None, "requires mmap") def _kill_with_event(self, event, name): tagname = "test_os_%s" % uuid.uuid1() m = mmap.mmap(-1, 1, tagname) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 6737068c0ff6d..e2da115501abe 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -13,7 +13,7 @@ from unittest import mock from test.support import import_helper -from test.support import is_emscripten +from test.support import is_emscripten, is_wasi from test.support import os_helper from test.support.os_helper import TESTFN, FakePath @@ -465,6 +465,9 @@ def test_parents_common(self): self.assertEqual(par[0], P('/a/b')) self.assertEqual(par[1], P('/a')) self.assertEqual(par[2], P('/')) + self.assertEqual(par[-1], P('/')) + self.assertEqual(par[-2], P('/a')) + self.assertEqual(par[-3], P('/a/b')) self.assertEqual(par[0:1], (P('/a/b'),)) self.assertEqual(par[:2], (P('/a/b'), P('/a'))) self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) @@ -472,6 +475,8 @@ def test_parents_common(self): self.assertEqual(par[::2], (P('/a/b'), P('/'))) self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) + with self.assertRaises(IndexError): + par[-4] with self.assertRaises(IndexError): par[3] @@ -1530,6 +1535,7 @@ def test_empty_path(self): p = self.cls('') self.assertEqual(p.stat(), os.stat('.')) + @unittest.skipIf(is_wasi, "WASI has no user accounts.") def test_expanduser_common(self): P = self.cls p = P('~') @@ -1693,10 +1699,15 @@ def _check(glob, expected): "dirA", "dirA/linkC", "dirB", "dirB/linkD", "dirC", "dirC/dirD", "dirE", "linkB", ]) + _check(p.rglob(""), ["", "dirA", "dirB", "dirC", "dirE", "dirC/dirD"]) p = P(BASE, "dirC") + _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) + _check(p.rglob("*/"), ["dirC/dirD"]) + _check(p.rglob(""), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) @@ -2369,6 +2380,9 @@ def test_is_socket_false(self): @unittest.skipIf( is_emscripten, "Unix sockets are not implemented on Emscripten." ) + @unittest.skipIf( + is_wasi, "Cannot create socket on WASI." + ) def test_is_socket_true(self): P = self.cls(BASE, 'mysock') sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -2525,7 +2539,8 @@ def _check_symlink_loop(self, *args, strict=True): print(path.resolve(strict)) @unittest.skipIf( - is_emscripten, "umask is not implemented on Emscripten." + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." ) def test_open_mode(self): old_mask = os.umask(0) @@ -2551,7 +2566,8 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten, "umask is not implemented on Emscripten." + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." ) def test_touch_mode(self): old_mask = os.umask(0) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 13b83a9db9eb3..d25bc112cfdc4 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -96,9 +96,7 @@ def setUp(self): self.skipTest("The %r command is not found" % cmd) self.old_cwd = os.getcwd() self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base) - change_cwd = os_helper.change_cwd(self.tmp_path) - change_cwd.__enter__() - self.addCleanup(change_cwd.__exit__, None, None, None) + self.enterContext(os_helper.change_cwd(self.tmp_path)) def tearDown(self): os.chdir(self.old_cwd) diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 7d542b5cfd783..02165a0244ddf 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -128,8 +128,7 @@ def test_poll2(self): cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, bufsize=0) - proc.__enter__() - self.addCleanup(proc.__exit__, None, None, None) + self.enterContext(proc) p = proc.stdout pollster = select.poll() pollster.register( p, select.POLLIN ) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index f44b8d0403ff2..28e5e90297e24 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -53,19 +53,13 @@ class PosixTester(unittest.TestCase): def setUp(self): # create empty file + self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "wb"): pass - self.teardown_files = [ os_helper.TESTFN ] - self._warnings_manager = warnings_helper.check_warnings() - self._warnings_manager.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.filterwarnings('ignore', '.* potential security risk .*', RuntimeWarning) - def tearDown(self): - for teardown_file in self.teardown_files: - os_helper.unlink(teardown_file) - self._warnings_manager.__exit__(None, None, None) - def testNoArgFunctions(self): # test posix functions which take no arguments and have # no side-effects which we need to cleanup (e.g., fork, wait, abort) @@ -973,8 +967,8 @@ def test_lchflags_symlink(self): self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) - self.teardown_files.append(_DUMMY_SYMLINK) dummy_symlink_st = os.lstat(_DUMMY_SYMLINK) def chflags_nofollow(path, flags): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 5fc4205beb125..97d3e9ea15bf3 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -387,8 +387,7 @@ def test_realpath_pardir(self): self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_basic(self): # Basic operation. @@ -398,8 +397,7 @@ def test_realpath_basic(self): finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_strict(self): # Bug #43757: raise FileNotFoundError in strict mode if we encounter @@ -411,8 +409,7 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_relative(self): try: @@ -421,8 +418,7 @@ def test_realpath_relative(self): finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): # Bug #930024, return the path unchanged if we get into an infinite @@ -463,8 +459,7 @@ def test_realpath_symlink_loops(self): os_helper.unlink(ABSTFN+"c") os_helper.unlink(ABSTFN+"a") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops_strict(self): # Bug #43757, raise OSError if we get into an infinite symlink loop in @@ -505,8 +500,7 @@ def test_realpath_symlink_loops_strict(self): os_helper.unlink(ABSTFN+"c") os_helper.unlink(ABSTFN+"a") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_repeated_indirect_symlinks(self): # Issue #6975. @@ -520,8 +514,7 @@ def test_realpath_repeated_indirect_symlinks(self): os_helper.unlink(ABSTFN + '/link') safe_rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_deep_recursion(self): depth = 10 @@ -540,8 +533,7 @@ def test_realpath_deep_recursion(self): os_helper.unlink(ABSTFN + '/%d' % i) safe_rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_resolve_parents(self): # We also need to resolve any symlinks in the parents of a relative @@ -560,8 +552,7 @@ def test_realpath_resolve_parents(self): safe_rmdir(ABSTFN + "/y") safe_rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_resolve_before_normalizing(self): # Bug #990669: Symbolic links should be resolved before we @@ -589,8 +580,7 @@ def test_realpath_resolve_before_normalizing(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_resolve_first(self): # Bug #1213894: The first component of the path, if not absolute, diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 13c77b6fa6822..ac181effe49bb 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -27,7 +27,8 @@ from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import (reap_children, captured_output, captured_stdout, - captured_stderr, is_emscripten, requires_docstrings) + captured_stderr, is_emscripten, is_wasi, + requires_docstrings) from test.support.os_helper import (TESTFN, rmtree, unlink) from test import pydoc_mod @@ -1356,7 +1357,10 @@ def a_fn_with_https_link(): ) -@unittest.skipIf(is_emscripten, "Socket server not available on Emscripten.") +@unittest.skipIf( + is_emscripten or is_wasi, + "Socket server not available on Emscripten/WASI." +) class PydocServerTest(unittest.TestCase): """Tests for pydoc._start_server""" diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 6e578458a2509..6f0441b66d9b8 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -12,7 +12,7 @@ from xml.parsers import expat from xml.parsers.expat import errors -from test.support import sortdict, is_emscripten +from test.support import sortdict, is_emscripten, is_wasi class SetAttributeTest(unittest.TestCase): @@ -469,6 +469,7 @@ def test_exception(self): if (sysconfig.is_python_build() and not (sys.platform == 'win32' and platform.machine() == 'ARM') and not is_emscripten + and not is_wasi ): self.assertIn('call_with_frame("StartElement"', entries[1][3]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index c1014753802c9..6d61412f16068 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,6 +1,6 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, - check_disallow_instantiation, is_emscripten) + check_disallow_instantiation, is_emscripten, is_wasi) import locale import re import string @@ -1974,7 +1974,10 @@ def test_bug_20998(self): # with ignore case. self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) - @unittest.skipIf(is_emscripten, "musl libc issue on Emscripten, bpo-46390") + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) def test_locale_caching(self): # Issue #22410 oldlocale = locale.setlocale(locale.LC_CTYPE) @@ -2011,7 +2014,10 @@ def check_en_US_utf8(self): self.assertIsNone(re.match(b'(?Li)\xc5', b'\xe5')) self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5')) - @unittest.skipIf(is_emscripten, "musl libc issue on Emscripten, bpo-46390") + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) def test_locale_compiled(self): oldlocale = locale.setlocale(locale.LC_CTYPE) self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) @@ -2411,6 +2417,30 @@ def test_bug_gh91616(self): self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer self.assertTrue(re.fullmatch(r'(?s:(?=(?P.*?\.))(?P=g0).*)\Z', "a.txt")) + def test_template_function_and_flag_is_deprecated(self): + with self.assertWarns(DeprecationWarning) as cm: + template_re1 = re.template(r'a') + self.assertIn('re.template()', str(cm.warning)) + self.assertIn('is deprecated', str(cm.warning)) + self.assertIn('function', str(cm.warning)) + self.assertNotIn('flag', str(cm.warning)) + + with self.assertWarns(DeprecationWarning) as cm: + # we deliberately use more flags here to test that that still + # triggers the warning + # if paranoid, we could test multiple different combinations, + # but it's probably not worth it + template_re2 = re.compile(r'a', flags=re.TEMPLATE|re.UNICODE) + self.assertIn('re.TEMPLATE', str(cm.warning)) + self.assertIn('is deprecated', str(cm.warning)) + self.assertIn('flag', str(cm.warning)) + self.assertNotIn('function', str(cm.warning)) + + # while deprecated, is should still function + self.assertEqual(template_re1, template_re2) + self.assertTrue(template_re1.match('ahoy')) + self.assertFalse(template_re1.match('nope')) + def get_debug_out(pat): with captured_stdout() as out: @@ -2605,11 +2635,11 @@ def test_flags_repr(self): "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000") self.assertEqual( repr(~re.I), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.DEBUG|0x1") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.TEMPLATE|re.DEBUG") self.assertEqual(repr(~(re.I|re.S|re.X)), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0x1") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG") self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))), - "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0xffe01") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG|0xffe00") class ImplementationTest(unittest.TestCase): diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 3821d66c2db7d..8d89e2a822445 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -308,8 +308,9 @@ def log_message(self, format, *args): pass -@unittest.skipIf( - support.is_emscripten, "Socket server not available on Emscripten." +@unittest.skipUnless( + support.has_socket_support, + "Socket server requires working socket." ) class PasswordProtectedSiteTestCase(unittest.TestCase): diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index c927331d438b0..c2db88c203920 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -19,8 +19,8 @@ resource = None -if support.is_emscripten: - raise unittest.SkipTest("Cannot create socketpair on Emscripten.") +if support.is_emscripten or support.is_wasi: + raise unittest.SkipTest("Cannot create socketpair on Emscripten/WASI.") if hasattr(socket, 'socketpair'): diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 3b57517a86101..43f23dbbf9bf7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1022,8 +1022,7 @@ def test_repr(self): class TestBasicOpsMixedStringBytes(TestBasicOps, unittest.TestCase): def setUp(self): - self._warning_filters = warnings_helper.check_warnings() - self._warning_filters.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.simplefilter('ignore', BytesWarning) self.case = "string and bytes set" self.values = ["a", "b", b"a", b"b"] @@ -1031,9 +1030,6 @@ def setUp(self): self.dup = set(self.values) self.length = 4 - def tearDown(self): - self._warning_filters.__exit__(None, None, None) - def test_repr(self): self.check_repr_against_values() diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 7003386345280..c94390589af3e 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1294,6 +1294,10 @@ def test_copyfile_same_file(self): self.assertEqual(read_file(src_file), 'foo') @unittest.skipIf(MACOS or SOLARIS or _winapi, 'On MACOS, Solaris and Windows the errors are not confusing (though different)') + # gh-92670: The test uses a trailing slash to force the OS consider + # the path as a directory, but on AIX the trailing slash has no effect + # and is considered as a file. + @unittest.skipIf(AIX, 'Not valid on AIX, see gh-92670') def test_copyfile_nonexistent_dir(self): # Issue 43219 src_dir = self.mkdtemp() @@ -2648,6 +2652,7 @@ def test_stty_match(self): self.assertEqual(expected, actual) + @unittest.skipIf(support.is_wasi, "WASI has no /dev/null") def test_fallback(self): with os_helper.EnvironmentVarGuard() as env: del env['LINES'] diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 2ba29fc837d44..6aa529b062000 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -107,6 +107,10 @@ def test_interprocess_signal(self): script = os.path.join(dirname, 'signalinterproctester.py') assert_python_ok(script) + @unittest.skipUnless( + hasattr(signal, "valid_signals"), + "requires signal.valid_signals" + ) def test_valid_signals(self): s = signal.valid_signals() self.assertIsInstance(s, set) @@ -212,6 +216,7 @@ def test_invalid_fd(self): self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) + @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_invalid_socket(self): sock = socket.socket() fd = sock.fileno() @@ -222,6 +227,7 @@ def test_invalid_socket(self): # Emscripten does not support fstat on pipes yet. # https://github.com/emscripten-core/emscripten/issues/16414 @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_result(self): r1, w1 = os.pipe() self.addCleanup(os.close, r1) @@ -240,6 +246,7 @@ def test_set_wakeup_fd_result(self): self.assertEqual(signal.set_wakeup_fd(-1), -1) @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_set_wakeup_fd_socket_result(self): sock1 = socket.socket() self.addCleanup(sock1.close) @@ -260,6 +267,7 @@ def test_set_wakeup_fd_socket_result(self): # function to test if a socket is in non-blocking mode. @unittest.skipIf(sys.platform == "win32", "tests specific to POSIX") @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_blocking(self): rfd, wfd = os.pipe() self.addCleanup(os.close, rfd) @@ -320,6 +328,7 @@ def check_signum(signals): assert_python_ok('-c', code) @unittest.skipIf(_testcapi is None, 'need _testcapi') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_wakeup_write_error(self): # Issue #16105: write() errors in the C signal handler should not # pass silently. @@ -659,6 +668,7 @@ def handler(signum, frame): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") @unittest.skipUnless(hasattr(signal, 'siginterrupt'), "needs signal.siginterrupt()") @support.requires_subprocess() +@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): def readpipe_interrupted(self, interrupt): diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py index 57eb98ebc1d1e..39ff8793648ba 100644 --- a/Lib/test/test_smtpd.py +++ b/Lib/test/test_smtpd.py @@ -10,6 +10,9 @@ smtpd = warnings_helper.import_deprecated('smtpd') asyncore = warnings_helper.import_deprecated('asyncore') +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + class DummyServer(smtpd.SMTPServer): def __init__(self, *args, **kwargs): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py old mode 100755 new mode 100644 index 613363722cf02..1aaa9e44f90c6 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -338,9 +338,7 @@ def serverExplicitReady(self): self.server_ready.set() def _setUp(self): - self.wait_threads = threading_helper.wait_threads_exit() - self.wait_threads.__enter__() - self.addCleanup(self.wait_threads.__exit__, None, None, None) + self.enterContext(threading_helper.wait_threads_exit()) self.server_ready = threading.Event() self.client_ready = threading.Event() diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index e132fcdfb0e65..e514adb35eec1 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -21,33 +21,19 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import os import sqlite3 as sqlite import subprocess import sys import threading import unittest +import urllib.parse -from test.support import ( - SHORT_TIMEOUT, - bigmemtest, - check_disallow_instantiation, - threading_helper, -) +from test.support import SHORT_TIMEOUT, bigmemtest, check_disallow_instantiation +from test.support import threading_helper from _testcapi import INT_MAX, ULLONG_MAX from os import SEEK_SET, SEEK_CUR, SEEK_END -from test.support.os_helper import TESTFN, unlink, temp_dir - - -# Helper for tests using TESTFN -@contextlib.contextmanager -def managed_connect(*args, in_mem=False, **kwargs): - cx = sqlite.connect(*args, **kwargs) - try: - yield cx - finally: - cx.close() - if not in_mem: - unlink(TESTFN) +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath # Helper for temporary memory databases @@ -242,7 +228,7 @@ def test_module_constants(self): "SQLITE_READONLY_CANTLOCK", "SQLITE_READONLY_RECOVERY", ] - if sqlite.version_info >= (3, 7, 16): + if sqlite.sqlite_version_info >= (3, 7, 16): consts += [ "SQLITE_CONSTRAINT_CHECK", "SQLITE_CONSTRAINT_COMMITHOOK", @@ -255,63 +241,63 @@ def test_module_constants(self): "SQLITE_CONSTRAINT_VTAB", "SQLITE_READONLY_ROLLBACK", ] - if sqlite.version_info >= (3, 7, 17): + if sqlite.sqlite_version_info >= (3, 7, 17): consts += [ "SQLITE_IOERR_MMAP", "SQLITE_NOTICE_RECOVER_ROLLBACK", "SQLITE_NOTICE_RECOVER_WAL", ] - if sqlite.version_info >= (3, 8, 0): + if sqlite.sqlite_version_info >= (3, 8, 0): consts += [ "SQLITE_BUSY_SNAPSHOT", "SQLITE_IOERR_GETTEMPPATH", "SQLITE_WARNING_AUTOINDEX", ] - if sqlite.version_info >= (3, 8, 1): + if sqlite.sqlite_version_info >= (3, 8, 1): consts += ["SQLITE_CANTOPEN_CONVPATH", "SQLITE_IOERR_CONVPATH"] - if sqlite.version_info >= (3, 8, 2): + if sqlite.sqlite_version_info >= (3, 8, 2): consts.append("SQLITE_CONSTRAINT_ROWID") - if sqlite.version_info >= (3, 8, 3): + if sqlite.sqlite_version_info >= (3, 8, 3): consts.append("SQLITE_READONLY_DBMOVED") - if sqlite.version_info >= (3, 8, 7): + if sqlite.sqlite_version_info >= (3, 8, 7): consts.append("SQLITE_AUTH_USER") - if sqlite.version_info >= (3, 9, 0): + if sqlite.sqlite_version_info >= (3, 9, 0): consts.append("SQLITE_IOERR_VNODE") - if sqlite.version_info >= (3, 10, 0): + if sqlite.sqlite_version_info >= (3, 10, 0): consts.append("SQLITE_IOERR_AUTH") - if sqlite.version_info >= (3, 14, 1): + if sqlite.sqlite_version_info >= (3, 14, 1): consts.append("SQLITE_OK_LOAD_PERMANENTLY") - if sqlite.version_info >= (3, 21, 0): + if sqlite.sqlite_version_info >= (3, 21, 0): consts += [ "SQLITE_IOERR_BEGIN_ATOMIC", "SQLITE_IOERR_COMMIT_ATOMIC", "SQLITE_IOERR_ROLLBACK_ATOMIC", ] - if sqlite.version_info >= (3, 22, 0): + if sqlite.sqlite_version_info >= (3, 22, 0): consts += [ "SQLITE_ERROR_MISSING_COLLSEQ", "SQLITE_ERROR_RETRY", "SQLITE_READONLY_CANTINIT", "SQLITE_READONLY_DIRECTORY", ] - if sqlite.version_info >= (3, 24, 0): + if sqlite.sqlite_version_info >= (3, 24, 0): consts += ["SQLITE_CORRUPT_SEQUENCE", "SQLITE_LOCKED_VTAB"] - if sqlite.version_info >= (3, 25, 0): + if sqlite.sqlite_version_info >= (3, 25, 0): consts += ["SQLITE_CANTOPEN_DIRTYWAL", "SQLITE_ERROR_SNAPSHOT"] - if sqlite.version_info >= (3, 31, 0): + if sqlite.sqlite_version_info >= (3, 31, 0): consts += [ "SQLITE_CANTOPEN_SYMLINK", "SQLITE_CONSTRAINT_PINNED", "SQLITE_OK_SYMLINK", ] - if sqlite.version_info >= (3, 32, 0): + if sqlite.sqlite_version_info >= (3, 32, 0): consts += [ "SQLITE_BUSY_TIMEOUT", "SQLITE_CORRUPT_INDEX", "SQLITE_IOERR_DATA", ] - if sqlite.version_info >= (3, 34, 0): - const.append("SQLITE_IOERR_CORRUPTFS") + if sqlite.sqlite_version_info >= (3, 34, 0): + consts.append("SQLITE_IOERR_CORRUPTFS") for const in consts: with self.subTest(const=const): self.assertTrue(hasattr(sqlite, const)) @@ -333,7 +319,7 @@ def test_error_code_on_exception(self): @unittest.skipIf(sqlite.sqlite_version_info <= (3, 7, 16), "Requires SQLite 3.7.16 or newer") def test_extended_error_code_on_exception(self): - with managed_connect(":memory:", in_mem=True) as con: + with memory_database() as con: with con: con.execute("create table t(t integer check(t > 0))") errmsg = "constraint failed" @@ -400,7 +386,7 @@ def test_cursor(self): def test_failed_open(self): YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db" with self.assertRaises(sqlite.OperationalError): - con = sqlite.connect(YOU_CANNOT_OPEN_THIS) + sqlite.connect(YOU_CANNOT_OPEN_THIS) def test_close(self): self.cx.close() @@ -663,24 +649,84 @@ class OpenTests(unittest.TestCase): def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ - class Path: - def __fspath__(self): - return TESTFN - path = Path() - with managed_connect(path) as cx: + path = FakePath(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") + @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") + def test_open_with_undecodable_path(self): + path = TESTFN_UNDECODABLE + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) cx.execute(self._sql) def test_open_uri(self): - with managed_connect(TESTFN) as cx: + path = TESTFN + self.addCleanup(unlink, path) + uri = "file:" + urllib.parse.quote(os.fsencode(path)) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: + self.assertTrue(os.path.exists(path)) cx.execute(self._sql) - with managed_connect(f"file:{TESTFN}", uri=True) as cx: + + def test_open_unquoted_uri(self): + path = TESTFN + self.addCleanup(unlink, path) + uri = "file:" + path + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: + self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + + def test_open_uri_readonly(self): + path = TESTFN + self.addCleanup(unlink, path) + uri = "file:" + urllib.parse.quote(os.fsencode(path)) + "?mode=ro" + self.assertFalse(os.path.exists(path)) + # Cannot create new DB with self.assertRaises(sqlite.OperationalError): - with managed_connect(f"file:{TESTFN}?mode=ro", uri=True) as cx: + sqlite.connect(uri, uri=True) + self.assertFalse(os.path.exists(path)) + sqlite.connect(path).close() + self.assertTrue(os.path.exists(path)) + # Cannot modify new DB + with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: + with self.assertRaises(sqlite.OperationalError): cx.execute(self._sql) + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") + @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") + @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") + def test_open_undecodable_uri(self): + path = TESTFN_UNDECODABLE + self.addCleanup(unlink, path) + uri = "file:" + urllib.parse.quote(path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + + def test_factory_database_arg(self): + def factory(database, *args, **kwargs): + nonlocal database_arg + database_arg = database + return sqlite.Connection(":memory:", *args, **kwargs) + + for database in (TESTFN, os.fsencode(TESTFN), + FakePath(TESTFN), FakePath(os.fsencode(TESTFN))): + database_arg = None + sqlite.connect(database, factory=factory).close() + self.assertEqual(database_arg, database) + def test_database_keyword(self): - with sqlite.connect(database=":memory:") as cx: + with contextlib.closing(sqlite.connect(database=":memory:")) as cx: self.assertEqual(type(cx), sqlite.Connection) diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 19bb84bf38a36..0b727cecb0e8c 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,7 +28,7 @@ from test import support from unittest.mock import patch -from test.test_sqlite3.test_dbapi import memory_database, managed_connect, cx_limit +from test.test_sqlite3.test_dbapi import memory_database, cx_limit class RegressionTests(unittest.TestCase): @@ -422,7 +422,7 @@ def test_return_empty_bytestring(self): self.assertEqual(val, b'') def test_table_lock_cursor_replace_stmt(self): - with managed_connect(":memory:", in_mem=True) as con: + with memory_database() as con: cur = con.cursor() cur.execute("create table t(t)") cur.executemany("insert into t values(?)", @@ -433,7 +433,7 @@ def test_table_lock_cursor_replace_stmt(self): con.commit() def test_table_lock_cursor_dealloc(self): - with managed_connect(":memory:", in_mem=True) as con: + with memory_database() as con: con.execute("create table t(t)") con.executemany("insert into t values(?)", ((v,) for v in range(5))) @@ -444,7 +444,7 @@ def test_table_lock_cursor_dealloc(self): con.commit() def test_table_lock_cursor_non_readonly_select(self): - with managed_connect(":memory:", in_mem=True) as con: + with memory_database() as con: con.execute("create table t(t)") con.executemany("insert into t values(?)", ((v,) for v in range(5))) @@ -459,7 +459,7 @@ def dup(v): con.commit() def test_executescript_step_through_select(self): - with managed_connect(":memory:", in_mem=True) as con: + with memory_database() as con: values = [(v,) for v in range(5)] with con: con.execute("create table t(t)") diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 0eb8d18b3561e..fed76378726c9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1999,9 +1999,8 @@ def setUp(self): self.server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) self.server_context.load_cert_chain(SIGNED_CERTFILE) server = ThreadedEchoServer(context=self.server_context) + self.enterContext(server) self.server_addr = (HOST, server.port) - server.__enter__() - self.addCleanup(server.__exit__, None, None, None) def test_connect(self): with test_wrap_socket(socket.socket(socket.AF_INET), @@ -3713,8 +3712,7 @@ def _recvfrom_into(): def test_recv_zero(self): server = ThreadedEchoServer(CERTFILE) - server.__enter__() - self.addCleanup(server.__exit__, None, None) + self.enterContext(server) s = socket.create_connection((HOST, server.port)) self.addCleanup(s.close) s = test_wrap_socket(s, suppress_ragged_eofs=False) diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 2e1e2c349c8d0..193a0fc15d9bc 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -161,7 +161,7 @@ def test_directory(self): else: self.assertEqual(modestr[0], 'd') - @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') + @os_helper.skip_unless_symlink def test_link(self): try: os.symlink(os.getcwd(), TESTFN) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index ed6021d60bde7..6de98241c294d 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1742,6 +1742,12 @@ def test_repeated_single_value(self): data = [x]*count self.assertEqual(self.func(data), float(x)) + def test_single_value(self): + # Override method from AverageMixin. + # Average of a single value is the value as a float. + for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): + self.assertEqual(self.func([x]), float(x)) + def test_odd_fractions(self): # Test median_grouped works with an odd number of Fractions. F = Fraction diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index dce49809385c6..23bcceedd71b2 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -664,6 +664,7 @@ def id(self): self.assertTrue(support.match_test(test_chdir)) @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") + @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): # We cannot test the absolute value of fd_count(): on old Linux # kernel or glibc versions, os.urandom() keeps a FD open on @@ -691,7 +692,7 @@ def test_print_warning(self): 'Warning -- a\nWarning -- b\n') def test_has_strftime_extensions(self): - if support.is_emscripten or support.is_wasi or sys.platform == "win32": + if support.is_emscripten or sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index b1c8f6f80af83..162fd8328582c 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,6 +7,7 @@ import gc from functools import wraps import asyncio +from test.support import import_helper support.requires_working_socket(module=True) @@ -608,6 +609,58 @@ def run(tracer): self.compare_events(doit_async.__code__.co_firstlineno, tracer.events, events) + def test_async_for_backwards_jump_has_no_line(self): + async def arange(n): + for i in range(n): + yield i + async def f(): + async for i in arange(3): + if i > 100: + break # should never be traced + + tracer = self.make_tracer() + coro = f() + try: + sys.settrace(tracer.trace) + coro.send(None) + except Exception: + pass + finally: + sys.settrace(None) + + events = [ + (0, 'call'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-1, 'line'), + (-1, 'return'), + (1, 'exception'), + (2, 'line'), + (1, 'line'), + (-1, 'call'), + (-2, 'line'), + (-1, 'line'), + (-1, 'return'), + (1, 'exception'), + (2, 'line'), + (1, 'line'), + (-1, 'call'), + (-2, 'line'), + (-1, 'line'), + (-1, 'return'), + (1, 'exception'), + (2, 'line'), + (1, 'line'), + (-1, 'call'), + (-2, 'line'), + (-2, 'return'), + (1, 'exception'), + (1, 'return'), + ] + self.compare_events(f.__code__.co_firstlineno, + tracer.events, events) + def test_21_repeated_pass(self): def func(): pass @@ -1473,6 +1526,58 @@ def __init__(self): (3, 'return'), (1, 'return')]) + @support.cpython_only + def test_no_line_event_after_creating_generator(self): + # Spurious line events before call events only show up with C tracer + + # Skip this test if the _testcapi module isn't available. + _testcapi = import_helper.import_module('_testcapi') + + def gen(): + yield 1 + + def func(): + for _ in ( + gen() + ): + pass + + EXPECTED_EVENTS = [ + (0, 'call'), + (2, 'line'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (4, 'line'), + (1, 'line'), + (-2, 'call'), + (-2, 'return'), + (1, 'return'), + ] + + # C level events should be the same as expected and the same as Python level. + + events = [] + # Turning on and off tracing must be on same line to avoid unwanted LINE events. + _testcapi.settrace_to_record(events); func(); sys.settrace(None) + start_line = func.__code__.co_firstlineno + events = [ + (line-start_line, EVENT_NAMES[what]) + for (what, line, arg) in events + ] + self.assertEqual(events, EXPECTED_EVENTS) + + self.run_and_compare(func, EXPECTED_EVENTS) + + +EVENT_NAMES = [ + 'call', + 'exception', + 'line', + 'return' +] + class SkipLineEventsTraceTestCase(TraceTestCase): """Repeat the trace tests, but with per-line events skipped""" @@ -2431,6 +2536,54 @@ def gen(): next(gen()) output.append(5) + @jump_test(2, 3, [1, 3]) + def test_jump_forward_over_listcomp(output): + output.append(1) + x = [i for i in range(10)] + output.append(3) + + # checking for segfaults. + # See https://github.com/python/cpython/issues/92311 + @jump_test(3, 1, []) + def test_jump_backward_over_listcomp(output): + a = 1 + x = [i for i in range(10)] + c = 3 + + @jump_test(8, 2, [2, 7, 2]) + def test_jump_backward_over_listcomp_v2(output): + flag = False + output.append(2) + if flag: + return + x = [i for i in range(5)] + flag = 6 + output.append(7) + output.append(8) + + @async_jump_test(2, 3, [1, 3]) + async def test_jump_forward_over_async_listcomp(output): + output.append(1) + x = [i async for i in asynciter(range(10))] + output.append(3) + + @async_jump_test(3, 1, []) + async def test_jump_backward_over_async_listcomp(output): + a = 1 + x = [i async for i in asynciter(range(10))] + c = 3 + + @async_jump_test(8, 2, [2, 7, 2]) + async def test_jump_backward_over_async_listcomp_v2(output): + flag = False + output.append(2) + if flag: + return + x = [i async for i in asynciter(range(5))] + flag = 6 + output.append(7) + output.append(8) + class TestExtendedArgs(unittest.TestCase): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 12850cd635e99..a364043d3d9dd 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1498,7 +1498,10 @@ def test_stream_padding(self): @unittest.skipUnless(sys.platform != "win32" and hasattr(os, "umask"), "Missing umask implementation") - @unittest.skipIf(support.is_emscripten, "Emscripten's umask is a stub.") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "Emscripten's/WASI's umask is a stub." + ) def test_file_mode(self): # Test for issue #8464: Create files with correct # permissions. diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a05f3c84ccfc9..f056e5ccb17f9 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -90,14 +90,10 @@ class BaseTestCase(unittest.TestCase): b_check = re.compile(br"^[a-z0-9_-]{8}$") def setUp(self): - self._warnings_manager = warnings_helper.check_warnings() - self._warnings_manager.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.filterwarnings("ignore", category=RuntimeWarning, message="mktemp", module=__name__) - def tearDown(self): - self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f7dea136a87c0..9c6561c099f57 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -149,6 +149,7 @@ def test_args_argument(self): with self.subTest(target=target, args=args): t = threading.Thread(target=target, args=args) t.start() + t.join() @cpython_only def test_disallow_instantiation(self): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index dc0bbb0ee2931..884b14231f573 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -489,6 +489,9 @@ def test_monotonic(self): def test_perf_counter(self): time.perf_counter() + @unittest.skipIf( + support.is_wasi, "process_time not available on WASI" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 76fa5905fa49e..378db58f25594 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -6,6 +6,7 @@ import datetime from decimal import Decimal as D from pathlib import Path +import sys import tempfile import unittest @@ -91,11 +92,13 @@ def test_deepcopy(self): self.assertEqual(obj_copy, expected_obj) def test_inline_array_recursion_limit(self): - nest_count = 470 + # 470 with default recursion limit + nest_count = int(sys.getrecursionlimit() * 0.47) recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" tomllib.loads(recursive_array_toml) def test_inline_table_recursion_limit(self): - nest_count = 310 + # 310 with default recursion limit + nest_count = int(sys.getrecursionlimit() * 0.31) recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8399465f6052d..d6cd3d9bdd6a4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -603,22 +603,10 @@ class C(Generic[T]): pass ('generic[T]', '[int]', 'generic[int]'), ('generic[T]', '[int, str]', 'TypeError'), ('generic[T]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), - # Should raise TypeError: a) according to the tentative spec, - # unpacked types cannot be used as arguments to aliases that expect - # a fixed number of arguments; b) it's equivalent to generic[()]. - ('generic[T]', '[*tuple[()]]', 'generic[*tuple[()]]'), - ('generic[T]', '[*Tuple[()]]', 'TypeError'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to aliases that expect a fixed - # number of arguments. - ('generic[T]', '[*tuple[int]]', 'generic[*tuple[int]]'), - ('generic[T]', '[*Tuple[int]]', 'TypeError'), - # Ditto. - ('generic[T]', '[*tuple[int, str]]', 'generic[*tuple[int, str]]'), - ('generic[T]', '[*Tuple[int, str]]', 'TypeError'), - # Ditto. - ('generic[T]', '[*tuple[int, ...]]', 'generic[*tuple[int, ...]]'), - ('generic[T]', '[*Tuple[int, ...]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[T]', '[*tuple_type[()]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, ...]]', 'TypeError'), ('generic[T]', '[*Ts]', 'TypeError'), ('generic[T]', '[T, *Ts]', 'TypeError'), ('generic[T]', '[*Ts, T]', 'TypeError'), @@ -664,23 +652,29 @@ class C(Generic[T1, T2]): pass ('generic[T1, T2]', '[int, str]', 'generic[int, str]'), ('generic[T1, T2]', '[int, str, bool]', 'TypeError'), ('generic[T1, T2]', '[*tuple_type[int]]', 'TypeError'), - ('generic[T1, T2]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str]]', 'generic[int, str]'), ('generic[T1, T2]', '[*tuple_type[int, str, bool]]', 'TypeError'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to aliases that expect a fixed - # number of arguments. - ('generic[T1, T2]', '[*tuple[int, str], *tuple[float, bool]]', 'generic[*tuple[int, str], *tuple[float, bool]]'), - ('generic[T1, T2]', '[*Tuple[int, str], *Tuple[float, bool]]', 'TypeError'), + ('generic[T1, T2]', '[int, *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], str]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[()]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[()]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str, float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float, bool]]', 'TypeError'), ('generic[T1, T2]', '[tuple_type[int, ...]]', 'TypeError'), ('generic[T1, T2]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + # Should raise TypeError according to the tentative spec: unpacked + # types cannot be used as arguments to aliases that expect a fixed + # number of arguments. ('generic[T1, T2]', '[*tuple_type[int, ...]]', 'TypeError'), - - # Ditto. - ('generic[T1, T2]', '[*tuple[int, ...], *tuple[str, ...]]', 'generic[*tuple[int, ...], *tuple[str, ...]]'), - ('generic[T1, T2]', '[*Tuple[int, ...], *Tuple[str, ...]]', 'TypeError'), - + ('generic[T1, T2]', '[int, *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], str]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), ('generic[T1, T2]', '[*Ts]', 'TypeError'), ('generic[T1, T2]', '[T, *Ts]', 'TypeError'), ('generic[T1, T2]', '[*Ts, T]', 'TypeError'), @@ -720,7 +714,7 @@ class C(Generic[T1, T2, T3]): pass tests = [ # Alias # Args # Expected result ('generic[T1, bool, T2]', '[int, str]', 'generic[int, bool, str]'), - ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'generic[int, bool, str]'), ] for alias_template, args_template, expected_template in tests: @@ -753,96 +747,49 @@ class C(Generic[*Ts]): pass # Tuple because tuple currently behaves differently. tests = [ # Alias # Args # Expected result - ('C[*Ts]', '[()]', 'C[()]'), - ('tuple[*Ts]', '[()]', 'tuple[()]'), - ('Tuple[*Ts]', '[()]', 'Tuple[()]'), - - ('C[*Ts]', '[int]', 'C[int]'), - ('tuple[*Ts]', '[int]', 'tuple[int]'), - ('Tuple[*Ts]', '[int]', 'Tuple[int]'), - - ('C[*Ts]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'), - ('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'), - - ('C[*Ts]', '[*tuple_type[int]]', 'C[*tuple_type[int]]'), # Should be C[int] - ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[*tuple_type[int]]'), # Should be tuple[int] - ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[*tuple_type[int]]'), # Should be Tuple[int] - - ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*tuple_type[*Ts]]'), # Should be C[*Ts] - ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*tuple_type[*Ts]]'), # Should be tuple[*Ts] - ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*tuple_type[*Ts]]'), # Should be Tuple[*Ts] - - ('C[*Ts]', '[*tuple_type[int, str]]', 'C[*tuple_type[int, str]]'), # Should be C[int, str] - ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[*tuple_type[int, str]]'), # Should be tuple[int, str] - ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[*tuple_type[int, str]]'), # Should be Tuple[int, str] - - ('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'), - ('Tuple[*Ts]', '[tuple_type[int, ...]]', 'Tuple[tuple_type[int, ...]]'), - - ('C[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'C[tuple_type[int, ...], tuple_type[str, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'tuple[tuple_type[int, ...], tuple_type[str, ...]]'), - ('Tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'), - - ('C[*Ts]', '[*tuple_type[int, ...]]', 'C[*tuple_type[int, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[*tuple_type[int, ...]]'), - ('Tuple[*Ts]', '[*tuple_type[int, ...]]', 'Tuple[*tuple_type[int, ...]]'), + ('generic[*Ts]', '[()]', 'generic[()]'), + ('generic[*Ts]', '[int]', 'generic[int]'), + ('generic[*Ts]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'), + ('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'), # Technically, multiple unpackings are forbidden by PEP 646, but we # choose to be less restrictive at runtime, to allow folks room # to experiment. So all three of these should be valid. - ('C[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), - ('Tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), - - ('C[*Ts]', '[*Ts]', 'C[*Ts]'), - ('tuple[*Ts]', '[*Ts]', 'tuple[*Ts]'), - ('Tuple[*Ts]', '[*Ts]', 'Tuple[*Ts]'), - - ('C[*Ts]', '[T, *Ts]', 'C[T, *Ts]'), - ('tuple[*Ts]', '[T, *Ts]', 'tuple[T, *Ts]'), - ('Tuple[*Ts]', '[T, *Ts]', 'Tuple[T, *Ts]'), - - ('C[*Ts]', '[*Ts, T]', 'C[*Ts, T]'), - ('tuple[*Ts]', '[*Ts, T]', 'tuple[*Ts, T]'), - ('Tuple[*Ts]', '[*Ts, T]', 'Tuple[*Ts, T]'), - - ('C[T, *Ts]', '[int]', 'C[int]'), - ('tuple[T, *Ts]', '[int]', 'tuple[int]'), - ('Tuple[T, *Ts]', '[int]', 'Tuple[int]'), - - ('C[T, *Ts]', '[int, str]', 'C[int, str]'), - ('tuple[T, *Ts]', '[int, str]', 'tuple[int, str]'), - ('Tuple[T, *Ts]', '[int, str]', 'Tuple[int, str]'), - - ('C[T, *Ts]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'), - ('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'), - - ('C[T, *Ts]', '[*tuple[int, ...]]', 'C[*tuple[int, ...]]'), # Should be C[int, *tuple[int, ...]] - ('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto - ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'tuple[*tuple[int, ...]]'), # Should be tuple[int, *tuple[int, ...]] - ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]] - ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Should be Tuple[int, *tuple[int, ...]] - ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]] - - ('C[*Ts, T]', '[int]', 'C[int]'), - ('tuple[*Ts, T]', '[int]', 'tuple[int]'), - ('Tuple[*Ts, T]', '[int]', 'Tuple[int]'), - - ('C[*Ts, T]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts, T]', '[int, str]', 'tuple[int, str]'), - ('Tuple[*Ts, T]', '[int, str]', 'Tuple[int, str]'), - - ('C[*Ts, T]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[*Ts, T]', '[int, str, bool]', 'tuple[int, str, bool]'), - ('Tuple[*Ts, T]', '[int, str, bool]', 'Tuple[int, str, bool]'), + ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'), + + ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), + ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), + ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[int]', 'generic[int]'), + ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'), + ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), + ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), + + ('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]] + + ('generic[*Ts, T]', '[int]', 'generic[int]'), + ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'), + ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), + ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'), ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'), ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool, float]', 'TypeError'), + + ('generic[T1, *tuple_type[T2, ...]]', '[int, str]', 'generic[int, *tuple_type[str, ...]]'), + ('generic[*tuple_type[T1, ...], T2]', '[int, str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *tuple_type[generic[*Ts], ...]]', '[int, str, bool]', 'generic[int, *tuple_type[generic[str, bool], ...]]'), + ('generic[*tuple_type[generic[*Ts], ...], T1]', '[int, str, bool]', 'generic[*tuple_type[generic[int, str], ...], bool]'), ] for alias_template, args_template, expected_template in tests: @@ -7348,6 +7295,37 @@ def test_all_exported_names(self): self.assertSetEqual(computed_all, actual_all) +class TypeIterationTests(BaseTestCase): + _UNITERABLE_TYPES = ( + Any, + Union, + Union[str, int], + Union[str, T], + List, + Tuple, + Callable, + Callable[..., T], + Callable[[T], str], + Annotated, + Annotated[T, ''], + ) + + def test_cannot_iterate(self): + expected_error_regex = "object is not iterable" + for test_type in self._UNITERABLE_TYPES: + with self.subTest(type=test_type): + with self.assertRaisesRegex(TypeError, expected_error_regex): + iter(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + list(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + for _ in test_type: + pass + + def test_is_not_instance_of_iterable(self): + for type_to_test in self._UNITERABLE_TYPES: + self.assertNotIsInstance(type_to_test, collections.abc.Iterable) + if __name__ == '__main__': main() diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index 80c22c6cdd1da..fe25bfe9f88d1 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -110,7 +110,7 @@ def _test_single(self, filename): os.unlink(filename) self.assertTrue(not os.path.exists(filename)) # and again with os.open. - f = os.open(filename, os.O_CREAT) + f = os.open(filename, os.O_CREAT | os.O_WRONLY) os.close(f) try: self._do_single(filename) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index f999ae8c16cea..969aa16678f49 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -648,6 +648,9 @@ def test_star_expr_assign_target(self): self.check_src_roundtrip(source.format(target=target)) def test_star_expr_assign_target_multiple(self): + self.check_src_roundtrip("() = []") + self.check_src_roundtrip("[] = ()") + self.check_src_roundtrip("() = [a] = c, = [d] = e, f = () = g = h") self.check_src_roundtrip("a = b = c = d") self.check_src_roundtrip("a, b = c, d = e, f = g") self.check_src_roundtrip("[a, b] = [c, d] = [e, f] = g") diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 82f1d9dc2e7bb..f067560ca6caa 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -10,6 +10,7 @@ from unittest.mock import patch from test import support from test.support import os_helper +from test.support import socket_helper from test.support import warnings_helper import os try: @@ -24,6 +25,10 @@ import collections +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + def hexescape(char): """Escape char as RFC 2396 specifies""" hex_repr = hex(ord(char))[2:].upper() @@ -232,17 +237,12 @@ class ProxyTests(unittest.TestCase): def setUp(self): # Records changes to env vars - self.env = os_helper.EnvironmentVarGuard() + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) # Delete all proxy related env vars for k in list(os.environ): if 'proxy' in k.lower(): self.env.unset(k) - def tearDown(self): - # Restore all proxy related env vars - self.env.__exit__() - del self.env - def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.request.getproxies_environment() diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 04cfb492e4549..5da41c37bbfb8 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -3,6 +3,7 @@ from test import support from test.support import os_helper from test.support import socket_helper +from test.support import ResourceDenied from test.test_urllib2 import sanepathname2url import os diff --git a/Lib/test/test_urllib_response.py b/Lib/test/test_urllib_response.py index 73d2ef0424f4a..b76763f4ed824 100644 --- a/Lib/test/test_urllib_response.py +++ b/Lib/test/test_urllib_response.py @@ -4,6 +4,11 @@ import tempfile import urllib.response import unittest +from test import support + +if support.is_wasi: + raise unittest.SkipTest("Cannot create socket on WASI") + class TestResponse(unittest.TestCase): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d96cf1e6c7493..199160e4d0e72 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -8,6 +8,7 @@ import ensurepip import os import os.path +import pathlib import re import shutil import struct @@ -16,7 +17,7 @@ import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten) + requires_subprocess, is_emscripten, is_wasi) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -34,8 +35,8 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_emscripten: - raise unittest.SkipTest("venv is not available on Emscripten.") +if is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on Emscripten/WASI.") @requires_subprocess() def check_output(cmd, encoding=None): @@ -98,12 +99,23 @@ def isdir(self, *args): fn = self.get_env_file(*args) self.assertTrue(os.path.isdir(fn)) - def test_defaults(self): + def test_defaults_with_str_path(self): """ - Test the create function with default arguments. + Test the create function with default arguments and a str path. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) + self._check_output_of_default_create() + + def test_defaults_with_pathlib_path(self): + """ + Test the create function with default arguments and a pathlib.Path path. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self._check_output_of_default_create() + + def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) @@ -473,7 +485,9 @@ def test_pathsep_error(self): the path separator. """ rmtree(self.env_dir) - self.assertRaises(ValueError, venv.create, self.env_dir + os.pathsep) + bad_itempath = self.env_dir + os.pathsep + self.assertRaises(ValueError, venv.create, bad_itempath) + self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) @requireVenvCreate class EnsurePipTest(BaseTest): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index db25eaba1278f..aea77b192c100 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -10,7 +10,6 @@ import html import io import itertools -import locale import operator import os import pickle @@ -978,15 +977,13 @@ def test_tostring_xml_declaration(self): def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('') - preferredencoding = locale.getpreferredencoding() self.assertEqual( - f"\n", - ET.tostring(elem, encoding='unicode', xml_declaration=True) + ET.tostring(elem, encoding='unicode', xml_declaration=True), + "\n" ) def test_tostring_xml_declaration_cases(self): elem = ET.XML('ø') - preferredencoding = locale.getpreferredencoding() TESTCASES = [ # (expected_retval, encoding, xml_declaration) # ... xml_declaration = None @@ -1013,7 +1010,7 @@ def test_tostring_xml_declaration_cases(self): b"ø", 'US-ASCII', True), (b"\n" b"\xf8", 'ISO-8859-1', True), - (f"\n" + ("\n" "ø", 'unicode', True), ] @@ -1051,11 +1048,10 @@ def test_tostringlist_xml_declaration(self): b"\n" ) - preferredencoding = locale.getpreferredencoding() stringlist = ET.tostringlist(elem, encoding='unicode', xml_declaration=True) self.assertEqual( ''.join(stringlist), - f"\n" + "\n" ) self.assertRegex(stringlist[0], r"^<\?xml version='1.0' encoding='.+'?>") self.assertEqual(['', '', ''], stringlist[1:]) @@ -3740,17 +3736,16 @@ def test_write_to_filename_as_unicode(self): encoding = f.encoding os_helper.unlink(TESTFN) - try: - '\xf8'.encode(encoding) - except UnicodeEncodeError: - self.skipTest(f'default file encoding {encoding} not supported') - tree = ET.ElementTree(ET.XML('''\xf8''')) tree.write(TESTFN, encoding='unicode') with open(TESTFN, 'rb') as f: data = f.read() expected = "\xf8".encode(encoding, 'xmlcharrefreplace') - self.assertEqual(data, expected) + if encoding.lower() in ('utf-8', 'ascii'): + self.assertEqual(data, expected) + else: + self.assertIn(b"ø''') + self.assertEqual(f.read(), convlinesep( + b'''\n''' + b'''ø''')) with open(TESTFN, 'w', encoding='ISO-8859-1') as f: tree.write(f, encoding='unicode') self.assertFalse(f.closed) with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'''\xf8''') + self.assertEqual(f.read(), convlinesep( + b'''\n''' + b'''\xf8''')) def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 848bf4f76d453..f4c11d88c8a09 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1740,6 +1740,17 @@ def test_empty_file_raises_BadZipFile(self): fp.write("short file") self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) + def test_negative_central_directory_offset_raises_BadZipFile(self): + # Zip file containing an empty EOCD record + buffer = bytearray(b'PK\x05\x06' + b'\0'*18) + + # Set the size of the central directory bytes to become 1, + # causing the central directory offset to become negative + for dirsize in 1, 2**32-1: + buffer[12:16] = struct.pack(' 0: + waiter = waiters[0] + try: + waiter.release() + except RuntimeError: + # gh-92530: The previous call of notify() released the lock, + # but was interrupted before removing it from the queue. + # It can happen if a signal handler raises an exception, + # like CTRL+C which raises KeyboardInterrupt. + pass + else: + n -= 1 try: - all_waiters.remove(waiter) + waiters.remove(waiter) except ValueError: pass diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 3d23889c74f24..296320235afdd 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2126,7 +2126,7 @@ def wm_iconbitmap(self, bitmap=None, default=None): the bitmap if None is given. Under Windows, the DEFAULT parameter can be used to set the icon - for the widget and any descendents that don't have an icon set + for the widget and any descendants that don't have an icon set explicitly. DEFAULT can be the relative path to a .ico file (example: root.iconbitmap(default='myicon.ico') ). See Tk documentation for more information.""" @@ -2372,9 +2372,9 @@ def destroy(self): _default_root = None def readprofile(self, baseName, className): - """Internal function. It reads BASENAME.tcl and CLASSNAME.tcl into - the Tcl Interpreter and calls exec on the contents of BASENAME.py and - CLASSNAME.py if such a file exists in the home directory.""" + """Internal function. It reads .BASENAME.tcl and .CLASSNAME.tcl into + the Tcl Interpreter and calls exec on the contents of .BASENAME.py and + .CLASSNAME.py if such a file exists in the home directory.""" import os if 'HOME' in os.environ: home = os.environ['HOME'] else: home = os.curdir diff --git a/Lib/typing.py b/Lib/typing.py index bdc14e39033dc..40ab516f7c8ff 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -271,6 +271,16 @@ def _check_generic(cls, parameters, elen): raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" f" actual {alen}, expected {elen}") +def _unpack_args(args): + newargs = [] + for arg in args: + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs is not None and not (subargs and subargs[-1] is ...): + newargs.extend(subargs) + else: + newargs.append(arg) + return newargs + def _prepare_paramspec_params(cls, params): """Prepares the parameters for a Generic containing ParamSpec variables (internal helper). @@ -405,9 +415,25 @@ def __deepcopy__(self, memo): return self +class _NotIterable: + """Mixin to prevent iteration, without being compatible with Iterable. + + That is, we could do: + def __iter__(self): raise TypeError() + But this would make users of this mixin duck type-compatible with + collections.abc.Iterable - isinstance(foo, Iterable) would be True. + + Luckily, we can instead prevent iteration by setting __iter__ to None, which + is treated specially. + """ + + __slots__ = () + __iter__ = None + + # Internal indicator of special typing constructs. # See __doc__ instance attribute for specific docs. -class _SpecialForm(_Final, _root=True): +class _SpecialForm(_Final, _NotIterable, _root=True): __slots__ = ('_name', '__doc__', '_getitem') def __init__(self, getitem): @@ -877,12 +903,8 @@ def __repr__(self): def _is_unpacked_typevartuple(x: Any) -> bool: - return ( - isinstance(x, _UnpackGenericAlias) - # If x is Unpack[tuple[...]], __parameters__ will be empty. - and x.__parameters__ - and isinstance(x.__parameters__[0], TypeVarTuple) - ) + return ((not isinstance(x, type)) and + getattr(x, '__typing_is_unpacked_typevartuple__', False)) def _is_typevar_like(x: Any) -> bool: @@ -995,7 +1017,8 @@ def __init__(self, name, *constraints, bound=None, def __typing_subst__(self, arg): msg = "Parameters to generic types must be types." arg = _type_check(arg, msg, is_argument=True) - if (isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack): + if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or + (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): raise TypeError(f"{arg} is not valid as type argument") return arg @@ -1354,19 +1377,17 @@ def __getitem__(self, args): if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. raise TypeError(f"Cannot subscript already-subscripted {self}") + if not self.__parameters__: + raise TypeError(f"{self} is not a generic class") # Preprocess `args`. if not isinstance(args, tuple): args = (args,) args = tuple(_type_convert(p) for p in args) + args = _unpack_args(args) if (self._paramspec_tvars and any(isinstance(t, ParamSpec) for t in self.__parameters__)): args = _prepare_paramspec_params(self, args) - elif not any(isinstance(p, TypeVarTuple) for p in self.__parameters__): - # We only run this if there are no TypeVarTuples, because we - # don't check variadic generic arity at runtime (to reduce - # complexity of typing.py). - _check_generic(self, args, len(self.__parameters__)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) @@ -1390,16 +1411,28 @@ def _determine_new_args(self, args): params = self.__parameters__ # In the example above, this would be {T3: str} new_arg_by_param = {} + typevartuple_index = None for i, param in enumerate(params): if isinstance(param, TypeVarTuple): - j = len(args) - (len(params) - i - 1) - if j < i: - raise TypeError(f"Too few arguments for {self}") - new_arg_by_param.update(zip(params[:i], args[:i])) - new_arg_by_param[param] = args[i: j] - new_arg_by_param.update(zip(params[i + 1:], args[j:])) - break + if typevartuple_index is not None: + raise TypeError(f"More than one TypeVarTuple parameter in {self}") + typevartuple_index = i + + alen = len(args) + plen = len(params) + if typevartuple_index is not None: + i = typevartuple_index + j = alen - (plen - i - 1) + if j < i: + raise TypeError(f"Too few arguments for {self};" + f" actual {alen}, expected at least {plen-1}") + new_arg_by_param.update(zip(params[:i], args[:i])) + new_arg_by_param[params[i]] = tuple(args[i: j]) + new_arg_by_param.update(zip(params[i + 1:], args[j:])) else: + if alen != plen: + raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" + f" actual {alen}, expected {plen}") new_arg_by_param.update(zip(params, args)) new_args = [] @@ -1498,7 +1531,7 @@ def __iter__(self): # 1 for List and 2 for Dict. It may be -1 if variable number of # parameters are accepted (needs custom __getitem__). -class _SpecialGenericAlias(_BaseGenericAlias, _root=True): +class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): def __init__(self, origin, nparams, *, inst=True, name=None): if name is None: name = origin.__name__ @@ -1541,7 +1574,7 @@ def __or__(self, right): def __ror__(self, left): return Union[left, self] -class _CallableGenericAlias(_GenericAlias, _root=True): +class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' args = self.__args__ @@ -1606,7 +1639,7 @@ def __getitem__(self, params): return self.copy_with(params) -class _UnionGenericAlias(_GenericAlias, _root=True): +class _UnionGenericAlias(_NotIterable, _GenericAlias, _root=True): def copy_with(self, params): return Union[params] @@ -1707,14 +1740,25 @@ def __repr__(self): return '*' + repr(self.__args__[0]) def __getitem__(self, args): - if self.__typing_unpacked__(): + if self.__typing_is_unpacked_typevartuple__: return args return super().__getitem__(args) - def __typing_unpacked__(self): - # If x is Unpack[tuple[...]], __parameters__ will be empty. - return bool(self.__parameters__ and - isinstance(self.__parameters__[0], TypeVarTuple)) + @property + def __typing_unpacked_tuple_args__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + arg, = self.__args__ + if isinstance(arg, _GenericAlias): + assert arg.__origin__ is tuple + return arg.__args__ + return None + + @property + def __typing_is_unpacked_typevartuple__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + return isinstance(self.__args__[0], TypeVarTuple) class Generic: @@ -2046,7 +2090,7 @@ def _proto_hook(other): cls.__init__ = _no_init_or_replace_init -class _AnnotatedAlias(_GenericAlias, _root=True): +class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' @@ -3324,10 +3368,10 @@ def dataclass_transform( Example usage with a decorator function: - _T = TypeVar("_T") + T = TypeVar("T") @dataclass_transform() - def create_model(cls: type[_T]) -> type[_T]: + def create_model(cls: type[T]) -> type[T]: ... return cls @@ -3356,20 +3400,23 @@ class CustomerModel(ModelBase): id: int name: str - Each of the ``CustomerModel`` classes defined in this example will now - behave similarly to a dataclass created with the ``@dataclasses.dataclass`` - decorator. For example, the type checker will synthesize an ``__init__`` - method. + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + ``@dataclasses.dataclass``. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. The arguments to this decorator can be used to customize this behavior: - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be - True or False if it is omitted by the caller. + ``True`` or ``False`` if it is omitted by the caller. - ``order_default`` indicates whether the ``order`` parameter is assumed to be True or False if it is omitted by the caller. - ``kw_only_default`` indicates whether the ``kw_only`` parameter is assumed to be True or False if it is omitted by the caller. - ``field_specifiers`` specifies a static list of supported classes or functions that describe fields, similar to ``dataclasses.field()``. + - Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. At runtime, this decorator records its arguments in the ``__dataclass_transform__`` attribute on the decorated object. diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index eda951ce73e66..005d23f6d00ec 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -49,7 +49,7 @@ def testMultiply(self): 'defaultTestLoader', 'SkipTest', 'skip', 'skipIf', 'skipUnless', 'expectedFailure', 'TextTestResult', 'installHandler', 'registerResult', 'removeResult', 'removeHandler', - 'addModuleCleanup', 'doModuleCleanups'] + 'addModuleCleanup', 'doModuleCleanups', 'enterModuleContext'] # Expose obsolete functions for backwards compatibility # bpo-5846: Deprecated in Python 3.11, scheduled for removal in Python 3.13. @@ -59,7 +59,8 @@ def testMultiply(self): from .result import TestResult from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, - skipIf, skipUnless, expectedFailure, doModuleCleanups) + skipIf, skipUnless, expectedFailure, doModuleCleanups, + enterModuleContext) from .suite import BaseTestSuite, TestSuite from .loader import TestLoader, defaultTestLoader from .main import TestProgram, main diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 85b938fb293af..a90eed98f8714 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -58,6 +58,26 @@ def addAsyncCleanup(self, func, /, *args, **kwargs): # 3. Regular "def func()" that returns awaitable object self.addCleanup(*(func, *args), **kwargs) + async def enterAsyncContext(self, cm): + """Enters the supplied asynchronous context manager. + + If successful, also adds its __aexit__ method as a cleanup + function and returns the result of the __aenter__ method. + """ + # We look up the special methods on the type to match the with + # statement. + cls = type(cm) + try: + enter = cls.__aenter__ + exit = cls.__aexit__ + except AttributeError: + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the asynchronous context manager protocol" + ) from None + result = await enter(cm) + self.addAsyncCleanup(exit, cm, None, None, None) + return result + def _callSetUp(self): self._asyncioTestContext.run(self.setUp) self._callAsync(self.asyncSetUp) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 55770c06d7c5d..ffc8f19ddd38d 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -102,12 +102,31 @@ def _id(obj): return obj +def _enter_context(cm, addcleanup): + # We look up the special methods on the type to match the with + # statement. + cls = type(cm) + try: + enter = cls.__enter__ + exit = cls.__exit__ + except AttributeError: + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the context manager protocol") from None + result = enter(cm) + addcleanup(exit, cm, None, None, None) + return result + + _module_cleanups = [] def addModuleCleanup(function, /, *args, **kwargs): """Same as addCleanup, except the cleanup items are called even if setUpModule fails (unlike tearDownModule).""" _module_cleanups.append((function, args, kwargs)) +def enterModuleContext(cm): + """Same as enterContext, but module-wide.""" + return _enter_context(cm, addModuleCleanup) + def doModuleCleanups(): """Execute all module cleanup functions. Normally called for you after @@ -426,12 +445,25 @@ def addCleanup(self, function, /, *args, **kwargs): Cleanup items are called even if setUp fails (unlike tearDown).""" self._cleanups.append((function, args, kwargs)) + def enterContext(self, cm): + """Enters the supplied context manager. + + If successful, also adds its __exit__ method as a cleanup + function and returns the result of the __enter__ method. + """ + return _enter_context(cm, self.addCleanup) + @classmethod def addClassCleanup(cls, function, /, *args, **kwargs): """Same as addCleanup, except the cleanup items are called even if setUpClass fails (unlike tearDownClass).""" cls._class_cleanups.append((function, args, kwargs)) + @classmethod + def enterClassContext(cls, cm): + """Same as enterContext, but class-wide.""" + return _enter_context(cm, cls.addClassCleanup) + def setUp(self): "Hook method for setting up the test fixture before exercising it." pass diff --git a/Lib/unittest/test/test_async_case.py b/Lib/unittest/test/test_async_case.py index 1b910a44eea0d..beadcac070b43 100644 --- a/Lib/unittest/test/test_async_case.py +++ b/Lib/unittest/test/test_async_case.py @@ -14,6 +14,29 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) +class TestCM: + def __init__(self, ordering, enter_result=None): + self.ordering = ordering + self.enter_result = enter_result + + async def __aenter__(self): + self.ordering.append('enter') + return self.enter_result + + async def __aexit__(self, *exc_info): + self.ordering.append('exit') + + +class LacksEnterAndExit: + pass +class LacksEnter: + async def __aexit__(self, *exc_info): + pass +class LacksExit: + async def __aenter__(self): + pass + + VAR = contextvars.ContextVar('VAR', default=()) @@ -337,6 +360,36 @@ async def coro(): output = test.run() self.assertTrue(cancelled) + def test_enterAsyncContext(self): + events = [] + + class Test(unittest.IsolatedAsyncioTestCase): + async def test_func(slf): + slf.addAsyncCleanup(events.append, 'cleanup1') + cm = TestCM(events, 42) + self.assertEqual(await slf.enterAsyncContext(cm), 42) + slf.addAsyncCleanup(events.append, 'cleanup2') + events.append('test') + + test = Test('test_func') + output = test.run() + self.assertTrue(output.wasSuccessful(), output) + self.assertEqual(events, ['enter', 'test', 'cleanup2', 'exit', 'cleanup1']) + + def test_enterAsyncContext_arg_errors(self): + class Test(unittest.IsolatedAsyncioTestCase): + async def test_func(slf): + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await slf.enterAsyncContext(LacksEnterAndExit()) + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await slf.enterAsyncContext(LacksEnter()) + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await slf.enterAsyncContext(LacksExit()) + + test = Test('test_func') + output = test.run() + self.assertTrue(output.wasSuccessful()) + def test_debug_cleanup_same_loop(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index 18062ae5a5871..d3488b40e82bd 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -46,6 +46,29 @@ def cleanup(ordering, blowUp=False): raise Exception('CleanUpExc') +class TestCM: + def __init__(self, ordering, enter_result=None): + self.ordering = ordering + self.enter_result = enter_result + + def __enter__(self): + self.ordering.append('enter') + return self.enter_result + + def __exit__(self, *exc_info): + self.ordering.append('exit') + + +class LacksEnterAndExit: + pass +class LacksEnter: + def __exit__(self, *exc_info): + pass +class LacksExit: + def __enter__(self): + pass + + class TestCleanUp(unittest.TestCase): def testCleanUp(self): class TestableTest(unittest.TestCase): @@ -173,6 +196,39 @@ def cleanup2(): self.assertEqual(ordering, ['setUp', 'test', 'tearDown', 'cleanup1', 'cleanup2']) + def test_enterContext(self): + class TestableTest(unittest.TestCase): + def testNothing(self): + pass + + test = TestableTest('testNothing') + cleanups = [] + + test.addCleanup(cleanups.append, 'cleanup1') + cm = TestCM(cleanups, 42) + self.assertEqual(test.enterContext(cm), 42) + test.addCleanup(cleanups.append, 'cleanup2') + + self.assertTrue(test.doCleanups()) + self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1']) + + def test_enterContext_arg_errors(self): + class TestableTest(unittest.TestCase): + def testNothing(self): + pass + + test = TestableTest('testNothing') + + with self.assertRaisesRegex(TypeError, 'the context manager'): + test.enterContext(LacksEnterAndExit()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + test.enterContext(LacksEnter()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + test.enterContext(LacksExit()) + + self.assertEqual(test._cleanups, []) + + class TestClassCleanup(unittest.TestCase): def test_addClassCleanUp(self): class TestableTest(unittest.TestCase): @@ -451,6 +507,35 @@ def tearDownClass(cls): self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) + def test_enterClassContext(self): + class TestableTest(unittest.TestCase): + def testNothing(self): + pass + + cleanups = [] + + TestableTest.addClassCleanup(cleanups.append, 'cleanup1') + cm = TestCM(cleanups, 42) + self.assertEqual(TestableTest.enterClassContext(cm), 42) + TestableTest.addClassCleanup(cleanups.append, 'cleanup2') + + TestableTest.doClassCleanups() + self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1']) + + def test_enterClassContext_arg_errors(self): + class TestableTest(unittest.TestCase): + def testNothing(self): + pass + + with self.assertRaisesRegex(TypeError, 'the context manager'): + TestableTest.enterClassContext(LacksEnterAndExit()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + TestableTest.enterClassContext(LacksEnter()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + TestableTest.enterClassContext(LacksExit()) + + self.assertEqual(TestableTest._class_cleanups, []) + class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): @@ -1000,6 +1085,31 @@ def tearDown(self): 'cleanup2', 'setUp2', 'test2', 'tearDown2', 'cleanup3', 'tearDownModule', 'cleanup1']) + def test_enterModuleContext(self): + cleanups = [] + + unittest.addModuleCleanup(cleanups.append, 'cleanup1') + cm = TestCM(cleanups, 42) + self.assertEqual(unittest.enterModuleContext(cm), 42) + unittest.addModuleCleanup(cleanups.append, 'cleanup2') + + unittest.case.doModuleCleanups() + self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1']) + + def test_enterModuleContext_arg_errors(self): + class TestableTest(unittest.TestCase): + def testNothing(self): + pass + + with self.assertRaisesRegex(TypeError, 'the context manager'): + unittest.enterModuleContext(LacksEnterAndExit()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + unittest.enterModuleContext(LacksEnter()) + with self.assertRaisesRegex(TypeError, 'the context manager'): + unittest.enterModuleContext(LacksExit()) + + self.assertEqual(unittest.case._module_cleanups, []) + class Test_TextTestRunner(unittest.TestCase): """Tests for TextTestRunner.""" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 84997f268c930..6d580a434a7be 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -2679,22 +2679,26 @@ def getproxies_registry(): # Returned as Unicode but problems if not converted to ASCII proxyServer = str(winreg.QueryValueEx(internetSettings, 'ProxyServer')[0]) - if '=' in proxyServer: - # Per-protocol settings - for p in proxyServer.split(';'): - protocol, address = p.split('=', 1) - # See if address has a type:// prefix - if not re.match('(?:[^/:]+)://', address): - address = '%s://%s' % (protocol, address) - proxies[protocol] = address - else: - # Use one setting for all protocols - if proxyServer[:5] == 'http:': - proxies['http'] = proxyServer - else: - proxies['http'] = 'http://%s' % proxyServer - proxies['https'] = 'https://%s' % proxyServer - proxies['ftp'] = 'ftp://%s' % proxyServer + if '=' not in proxyServer and ';' not in proxyServer: + # Use one setting for all protocols. + proxyServer = 'http={0};https={0};ftp={0}'.format(proxyServer) + for p in proxyServer.split(';'): + protocol, address = p.split('=', 1) + # See if address has a type:// prefix + if not re.match('(?:[^/:]+)://', address): + # Add type:// prefix to address without specifying type + if protocol in ('http', 'https', 'ftp'): + # The default proxy type of Windows is HTTP + address = 'http://' + address + elif protocol == 'socks': + address = 'socks://' + address + proxies[protocol] = address + # Use SOCKS proxy for HTTP(S) protocols + if proxies.get('socks'): + # The default SOCKS proxy type of Windows is SOCKS4 + address = re.sub(r'^socks://', 'socks4://', proxies['socks']) + proxies['http'] = proxies.get('http') or address + proxies['https'] = proxies.get('https') or address internetSettings.Close() except (OSError, ValueError, TypeError): # Either registry key not found etc, or the value in an diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 7bfbadda7b497..6032f3648e15f 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -116,7 +116,7 @@ def create_if_needed(d): elif os.path.islink(d) or os.path.isfile(d): raise ValueError('Unable to create directory %r' % d) - if os.pathsep in env_dir: + if os.pathsep in os.fspath(env_dir): raise ValueError(f'Refusing to create a venv in {env_dir} because ' f'it contains the PATH separator {os.pathsep}.') if os.path.exists(env_dir) and self.clear: diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 5249c7ab82b84..a5cc65e789c00 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -728,16 +728,10 @@ def write(self, file_or_filename, encoding = "utf-8" else: encoding = "us-ascii" - enc_lower = encoding.lower() - with _get_writer(file_or_filename, enc_lower) as write: + with _get_writer(file_or_filename, encoding) as (write, declared_encoding): if method == "xml" and (xml_declaration or (xml_declaration is None and - enc_lower not in ("utf-8", "us-ascii", "unicode"))): - declared_encoding = encoding - if enc_lower == "unicode": - # Retrieve the default encoding for the xml declaration - import locale - declared_encoding = locale.getpreferredencoding() + declared_encoding.lower() not in ("utf-8", "us-ascii"))): write("\n" % ( declared_encoding,)) if method == "text": @@ -762,19 +756,20 @@ def _get_writer(file_or_filename, encoding): write = file_or_filename.write except AttributeError: # file_or_filename is a file name - if encoding == "unicode": - file = open(file_or_filename, "w") + if encoding.lower() == "unicode": + file = open(file_or_filename, "w", + errors="xmlcharrefreplace") else: file = open(file_or_filename, "w", encoding=encoding, errors="xmlcharrefreplace") with file: - yield file.write + yield file.write, file.encoding else: # file_or_filename is a file-like object # encoding determines if it is a text or binary writer - if encoding == "unicode": + if encoding.lower() == "unicode": # use a text writer as is - yield write + yield write, getattr(file_or_filename, "encoding", None) or "utf-8" else: # wrap a binary writer with TextIOWrapper with contextlib.ExitStack() as stack: @@ -805,7 +800,7 @@ def _get_writer(file_or_filename, encoding): # Keep the original file open when the TextIOWrapper is # destroyed stack.callback(file.detach) - yield file.write + yield file.write, encoding def _namespaces(elem, default_namespace=None): # identify namespaces used in this tree diff --git a/Lib/zipfile.py b/Lib/zipfile.py index dc02011084329..fe5c186deff34 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1381,6 +1381,8 @@ def _RealGetContents(self): print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat + if self.start_dir < 0: + raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index 8d699395f304c..a43ca9fc788a3 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2580 +{\rtf1\ansi\ansicpg1252\cocoartf2638 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.11.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.11.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index e6ccfcb3fce96..4ff8a0e874f20 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2580 +{\rtf1\ansi\ansicpg1252\cocoartf2638 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.11.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.11.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Makefile.pre.in b/Makefile.pre.in index e45d4fe3ecb6e..f4b23f6e76570 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1198,6 +1198,14 @@ regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py ############################################################################ # ABI +regen-abidump: all + @$(MKDIR_P) $(srcdir)/Doc/data/ + abidw "libpython$(LDVERSION).so" --no-architecture --out-file $(srcdir)/Doc/data/python$(LDVERSION).abi.new + @$(UPDATE_FILE) $(srcdir)/Doc/data/python$(LDVERSION).abi $(srcdir)/Doc/data/python$(LDVERSION).abi.new + +check-abidump: all + abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms + regen-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml diff --git a/Misc/ACKS b/Misc/ACKS index a55706d508a41..4ea1fa17d5400 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1062,6 +1062,7 @@ Robert Li Xuanji Li Zekun Li Zheao Li +Eli Libman Dan Lidral-Porter Robert van Liere Ross Light diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index a4ee9ae098bd9..803df6f51ce62 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -232,7 +232,7 @@ now result in :exc:`MemoryError`. Patch by Erlend E. Aasland. .. section: Library Fix segfault in :meth:`sqlite3.Connection.backup` if no argument was -provided. The regression was introduced by GH-23838. Patch by Erlend E. +provided. The regression was introduced by PR 23838. Patch by Erlend E. Aasland. .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 7e9cb77266bd9..286d0a8a7e919 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -574,7 +574,7 @@ raised. Patch by Erlend E. Aasland. .. nonce: t9XEkQ .. section: Library -Fix a regression introduced in GH-24562, where an empty bytestring was +Fix a regression introduced in PR 24562, where an empty bytestring was fetched as ``None`` instead of ``b''`` in :mod:`sqlite3`. Patch by Mariusz Felisiak. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 83ba504d04342..2a3d358edde90 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -941,7 +941,7 @@ result from ``entry_points()`` as deprecated. .. -.. bpo: 47383 +.. gh: 47383 .. date: 2021-04-08-19-32-26 .. nonce: YI1hdL .. section: Library @@ -1001,7 +1001,7 @@ some :mod:`dataclasses`. Fix :mod:`sqlite3` regression for zero-sized blobs with converters, where ``b""`` was returned instead of ``None``. The regression was introduced by -GH-24723. Patch by Erlend E. Aasland. +PR 24723. Patch by Erlend E. Aasland. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index fa30c693c34c1..33841d9e4e39b 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -1597,7 +1597,7 @@ use map function instead of genexpr in capwords. .. section: Library Fix typo: ``importlib.find_loader`` is really slated for removal in Python -3.12 not 3.10, like the others in GH-25169. +3.12 not 3.10, like the others in PR 25169. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 64c88c66ce50e..c135eff4598e4 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -570,7 +570,7 @@ planned). Patch by Alex Waygood. .. -.. bpo: 78157 +.. gh: 78157 .. date: 2022-05-05-20-40-45 .. nonce: IA_9na .. section: Library @@ -1288,7 +1288,7 @@ Deprecate the chunk module. .. -.. bpo: 91498 +.. gh: 91498 .. date: 2022-04-10-08-39-44 .. nonce: 8oII92 .. section: Library @@ -1373,6 +1373,7 @@ Suppress expression chaining for more :mod:`re` parsing errors. Remove undocumented and never working function ``re.template()`` and flag ``re.TEMPLATE``. +This was later reverted in 3.11.0b2 and deprecated instead. .. diff --git a/Misc/NEWS.d/3.11.0b2.rst b/Misc/NEWS.d/3.11.0b2.rst new file mode 100644 index 0000000000000..23014e39dd155 --- /dev/null +++ b/Misc/NEWS.d/3.11.0b2.rst @@ -0,0 +1,489 @@ +.. date: 2022-05-25-12-30-12 +.. gh-issue: 84694 +.. nonce: 5sjy2w +.. release date: 2022-05-30 +.. section: Core and Builtins + +The ``--experimental-isolated-subinterpreters`` configure option and +``EXPERIMENTAL_ISOLATED_SUBINTERPRETERS`` macro have been removed. + +.. + +.. date: 2022-05-25-04-07-22 +.. gh-issue: 91924 +.. nonce: -UyO4q +.. section: Core and Builtins + +Fix ``__lltrace__`` debug feature if the stdout encoding is not UTF-8. Patch +by Victor Stinner. + +.. + +.. date: 2022-05-22-02-37-50 +.. gh-issue: 93061 +.. nonce: r70Imp +.. section: Core and Builtins + +Backward jumps after ``async for`` loops are no longer given dubious line +numbers. + +.. + +.. date: 2022-05-21-23-21-37 +.. gh-issue: 93065 +.. nonce: 5I18WC +.. section: Core and Builtins + +Fix contextvars HAMT implementation to handle iteration over deep trees. + +The bug was discovered and fixed by Eli Libman. See +`MagicStack/immutables#84 +`_ for more details. + +.. + +.. date: 2022-05-15-15-25-05 +.. gh-issue: 90473 +.. nonce: MoPHYW +.. section: Core and Builtins + +Decrease default recursion limit on WASI to address limited call stack size. + +.. + +.. date: 2022-05-14-13-22-11 +.. gh-issue: 92804 +.. nonce: rAqpI2 +.. section: Core and Builtins + +Fix memory leak in ``memoryview`` iterator as it was not finalized at exit. +Patch by Kumar Aditya. + +.. + +.. date: 2022-05-12-13-23-19 +.. gh-issue: 92236 +.. nonce: sDRzUe +.. section: Core and Builtins + +Remove spurious "LINE" event when starting a generator or coroutine, visible +tracing functions implemented in C. + +.. + +.. date: 2022-05-10-11-34-35 +.. gh-issue: 92619 +.. nonce: u0V0lY +.. section: Core and Builtins + +Make the compiler duplicate an exit block only if none of its instructions +have a lineno (previously only the first instruction in the block was +checked, leading to unnecessarily duplicated blocks). + +.. + +.. date: 2022-05-03-20-12-18 +.. gh-issue: 92261 +.. nonce: aigLnb +.. section: Core and Builtins + +Fix hang when trying to iterate over a ``typing.Union``. + +.. + +.. date: 2022-05-27-13-18-18 +.. gh-issue: 93297 +.. nonce: e2zuHz +.. section: Library + +Make asyncio task groups prevent child tasks from being GCed + +.. + +.. date: 2022-05-25-02-45-41 +.. gh-issue: 90817 +.. nonce: yxANgU +.. section: Library + +The :func:`locale.resetlocale` function is deprecated and will be removed in +Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead. Patch by +Victor Stinner. + +.. + +.. date: 2022-05-24-10-59-02 +.. gh-issue: 92728 +.. nonce: zxTifq +.. section: Library + +The :func:`re.template` function and the corresponding :const:`re.TEMPLATE` +and :const:`re.T` flags are restored after they were removed in 3.11.0b1, +but they are now deprecated, so they might be removed from Python 3.13. + +.. + +.. date: 2022-05-21-13-16-16 +.. gh-issue: 93044 +.. nonce: eJ_XkZ +.. section: Library + +No longer convert the database argument of :func:`sqlite3.connect` to bytes +before passing it to the factory. + +.. + +.. date: 2022-05-20-15-52-43 +.. gh-issue: 93010 +.. nonce: WF-cAc +.. section: Library + +In a very special case, the email package tried to append the nonexistent +``InvalidHeaderError`` to the defect list. It should have been +``InvalidHeaderDefect``. + +.. + +.. date: 2022-05-19-13-33-18 +.. gh-issue: 92675 +.. nonce: ZeerMZ +.. section: Library + +Fix :func:`venv.ensure_directories` to accept :class:`pathlib.Path` +arguments in addition to :class:`str` paths. Patch by David Foster. + +.. + +.. date: 2022-05-18-21-04-09 +.. gh-issue: 87901 +.. nonce: lnf041 +.. section: Library + +Removed the ``encoding`` argument from :func:`os.popen` that was added in +3.11b1. + +.. + +.. date: 2022-05-18-17-18-41 +.. gh-issue: 91922 +.. nonce: DwWIsJ +.. section: Library + +Fix function :func:`sqlite.connect` and the :class:`sqlite.Connection` +constructor on non-UTF-8 locales. Also, they now support bytes paths +non-decodable with the current FS encoding. + +.. + +.. date: 2022-05-16-14-35-39 +.. gh-issue: 92839 +.. nonce: owSMyo +.. section: Library + +Fixed crash resulting from calling bisect.insort() or bisect.insort_left() +with the key argument not equal to None. + +.. + +.. date: 2022-05-14-11-41-23 +.. gh-issue: 90473 +.. nonce: kPdOZl +.. section: Library + +:mod:`subprocess` now fails early on Emscripten and WASI platforms to work +around missing :func:`os.pipe` on WASI. + +.. + +.. date: 2022-05-11-19-33-27 +.. gh-issue: 92671 +.. nonce: KE4v6a +.. section: Library + +Fixed :func:`ast.unparse` for empty tuples in the assignment target context. + +.. + +.. date: 2022-05-11-14-34-09 +.. gh-issue: 91581 +.. nonce: glkou2 +.. section: Library + +:meth:`~datetime.datetime.utcfromtimestamp` no longer attempts to resolve +``fold`` in the pure Python implementation, since the fold is never 1 in +UTC. In addition to being slightly faster in the common case, this also +prevents some errors when the timestamp is close to :attr:`datetime.min +`. Patch by Paul Ganssle. + +.. + +.. date: 2022-05-10-07-57-27 +.. gh-issue: 92550 +.. nonce: Rk_UzM +.. section: Library + +Fix :meth:`pathlib.Path.rglob` for empty pattern. + +.. + +.. date: 2022-05-09-09-28-02 +.. gh-issue: 92530 +.. nonce: M4Q1RS +.. section: Library + +Fix an issue that occurred after interrupting +:func:`threading.Condition.notify`. + +.. + +.. date: 2022-05-09-01-27-25 +.. gh-issue: 92531 +.. nonce: vV7S_O +.. section: Library + +The statistics.median_grouped() function now always return a float. +Formerly, it did not convert the input type when for sequences of length +one. + +.. + +.. date: 2022-04-25-10-23-01 +.. gh-issue: 91810 +.. nonce: DOHa6B +.. section: Library + +:class:`~xml.etree.ElementTree.ElementTree` method +:meth:`~xml.etree.ElementTree.ElementTree.write` and function +:func:`~xml.etree.ElementTree.tostring` now use the text file's encoding +("UTF-8" if not available) instead of locale encoding in XML declaration +when ``encoding="unicode"`` is specified. + +.. + +.. date: 2022-04-15-22-07-36 +.. gh-issue: 90622 +.. nonce: 0C6l8h +.. section: Library + +Worker processes for :class:`concurrent.futures.ProcessPoolExecutor` are no +longer spawned on demand (a feature added in 3.9) when the multiprocessing +context start method is ``"fork"`` as that can lead to deadlocks in the +child processes due to a fork happening while threads are running. + +.. + +.. date: 2022-04-15-13-16-25 +.. gh-issue: 91581 +.. nonce: 9OGsrN +.. section: Library + +Remove an unhandled error case in the C implementation of calls to +:meth:`datetime.fromtimestamp ` with no +time zone (i.e. getting a local time from an epoch timestamp). This should +have no user-facing effect other than giving a possibly more accurate error +message when called with timestamps that fall on 10000-01-01 in the local +time. Patch by Paul Ganssle. + +.. + +.. bpo: 39064 +.. date: 2022-04-03-19-40-09 +.. nonce: 76PbIz +.. section: Library + +:class:`zipfile.ZipFile` now raises :exc:`zipfile.BadZipFile` instead of +``ValueError`` when reading a corrupt zip file in which the central +directory offset is negative. + +.. + +.. bpo: 45393 +.. date: 2022-02-09-23-44-27 +.. nonce: 9v5Y8U +.. section: Library + +Fix the formatting for ``await x`` and ``not x`` in the operator precedence +table when using the :func:`help` system. + +.. + +.. bpo: 28249 +.. date: 2022-01-09-14-23-00 +.. nonce: 4dzB80 +.. section: Library + +Set :attr:`doctest.DocTest.lineno` to ``None`` when object does not have +:attr:`__doc__`. + +.. + +.. bpo: 45046 +.. date: 2021-08-29-19-59-16 +.. nonce: eGq0NC +.. section: Library + +Add support of context managers in :mod:`unittest`: methods +:meth:`~unittest.TestCase.enterContext` and +:meth:`~unittest.TestCase.enterClassContext` of class +:class:`~unittest.TestCase`, method +:meth:`~unittest.IsolatedAsyncioTestCase.enterAsyncContext` of class +:class:`~unittest.IsolatedAsyncioTestCase` and function +:func:`unittest.enterModuleContext`. + +.. + +.. bpo: 42627 +.. date: 2021-05-22-07-58-59 +.. nonce: EejtD0 +.. section: Library + +Fix incorrect parsing of Windows registry proxy settings + +.. + +.. date: 2022-05-26-11-33-23 +.. gh-issue: 86438 +.. nonce: kEGGmK +.. section: Documentation + +Clarify that :option:`-W` and :envvar:`PYTHONWARNINGS` are matched literally +and case-insensitively, rather than as regular expressions, in +:mod:`warnings`. + +.. + +.. date: 2022-05-18-23-58-26 +.. gh-issue: 92240 +.. nonce: bHvYiz +.. section: Documentation + +Added release dates for "What's New in Python 3.X" for 3.0, 3.1, 3.2, 3.8 +and 3.10 + +.. + +.. bpo: 40838 +.. date: 2022-01-13-16-03-15 +.. nonce: k3NVCf +.. section: Documentation + +Document that :func:`inspect.getdoc`, :func:`inspect.getmodule`, and +:func:`inspect.getsourcefile` might return ``None``. + +.. + +.. bpo: 38056 +.. date: 2019-09-12-08-28-17 +.. nonce: 6ktYkc +.. section: Documentation + +Overhaul the :ref:`error-handlers` documentation in :mod:`codecs`. + +.. + +.. bpo: 13553 +.. date: 2017-12-10-19-13-39 +.. nonce: gQbZs4 +.. section: Documentation + +Document tkinter.Tk args. + +.. + +.. date: 2022-05-12-05-51-06 +.. gh-issue: 92670 +.. nonce: 7L43Z_ +.. section: Tests + +Skip ``test_shutil.TestCopy.test_copyfile_nonexistent_dir`` test on AIX as +the test uses a trailing slash to force the OS consider the path as a +directory, but on AIX the trailing slash has no effect and is considered as +a file. + +.. + +.. date: 2022-05-12-10-19-15 +.. gh-issue: 90473 +.. nonce: -syvqK +.. section: Build + +Disable pymalloc and increase stack size on ``wasm32-wasi``. + +.. + +.. bpo: 34449 +.. date: 2018-08-21-11-10-18 +.. nonce: Z3qm3c +.. section: Build + +Drop invalid compiler switch ``-fPIC`` for HP aCC on HP-UX. Patch by Michael +Osipov. + +.. + +.. date: 2022-05-19-21-44-25 +.. gh-issue: 92817 +.. nonce: Jrf-Kv +.. section: Windows + +Ensures that :file:`py.exe` will prefer an active virtual environment over +default tags specified with environment variables or through a +:file:`py.ini` file. + +.. + +.. date: 2022-05-19-14-01-30 +.. gh-issue: 92984 +.. nonce: Dsxnlr +.. section: Windows + +Explicitly disable incremental linking for non-Debug builds + +.. + +.. date: 2022-05-16-11-45-06 +.. gh-issue: 92841 +.. nonce: NQx107 +.. section: Windows + +:mod:`asyncio` no longer throws ``RuntimeError: Event loop is closed`` on +interpreter exit after asynchronous socket activity. Patch by Oleg Iarygin. + +.. + +.. bpo: 46907 +.. date: 2022-05-05-06-27-59 +.. nonce: IW-uvT +.. section: Windows + +Update Windows installer to use SQLite 3.38.4. + +.. + +.. date: 2022-05-23-15-22-18 +.. gh-issue: 92898 +.. nonce: Qjc9d3 +.. section: C API + +Fix C++ compiler warnings when casting function arguments to ``PyObject*``. +Patch by Serge Guelton. + +.. + +.. date: 2022-05-19-18-05-51 +.. gh-issue: 92913 +.. nonce: Ass1Hv +.. section: C API + +Ensures changes to :c:member:`PyConfig.module_search_paths` are ignored +unless :c:member:`PyConfig.module_search_paths_set` is set + +.. + +.. date: 2022-05-13-18-17-48 +.. gh-issue: 92781 +.. nonce: TVDr3- +.. section: C API + +Avoid mixing declarations and code in the C API to fix the compiler warning: +"ISO C90 forbids mixed declarations and code" +[-Werror=declaration-after-statement]. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.11.0b3.rst b/Misc/NEWS.d/3.11.0b3.rst new file mode 100644 index 0000000000000..70195c25d6f80 --- /dev/null +++ b/Misc/NEWS.d/3.11.0b3.rst @@ -0,0 +1,27 @@ +.. date: 2022-05-30-19-00-38 +.. gh-issue: 93359 +.. nonce: zXV3A0 +.. release date: 2022-06-01 +.. section: Core and Builtins + +Ensure that custom :mod:`ast` nodes without explicit end positions can be +compiled. Patch by Pablo Galindo. + +.. + +.. date: 2022-05-30-10-22-46 +.. gh-issue: 93345 +.. nonce: gi1A4L +.. section: Core and Builtins + +Fix a crash in substitution of a ``TypeVar`` in nested generic alias after +``TypeVarTuple``. + +.. + +.. date: 2022-05-31-18-04-58 +.. gh-issue: 69093 +.. nonce: 6lSa0C +.. section: Build + +Fix ``Modules/Setup.stdlib.in`` rule for ``_sqlite3`` extension. diff --git a/Misc/NEWS.d/3.7.0a3.rst b/Misc/NEWS.d/3.7.0a3.rst index 067720efa516e..6576c1fadbff6 100644 --- a/Misc/NEWS.d/3.7.0a3.rst +++ b/Misc/NEWS.d/3.7.0a3.rst @@ -288,7 +288,7 @@ by Nir Soffer. .. -.. bpo: 321010 +.. bpo: 32101 .. date: 2017-11-29-00-42-47 .. nonce: -axD5l .. section: Library diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 5cd3fa32105c2..09b858d250c33 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -4617,7 +4617,7 @@ Based on patch by c-fos. .. section: Library Remove HMAC default to md5 marked for removal in 3.8 (removal originally -planned in 3.6, bump to 3.8 in gh-7062). +planned in 3.6, bump to 3.8 in PR 7062). .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index a9b6694c133f1..45f232f1948d5 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -1335,7 +1335,7 @@ module on POSIX systems. .. nonce: 9TWMlz .. section: Library -Revert GH-15522, which introduces a regression in +Revert PR 15522, which introduces a regression in :meth:`mimetypes.guess_type` due to improper handling of filenames as urls. .. diff --git a/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst b/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst new file mode 100644 index 0000000000000..f48ed37c81445 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-06-04-13-15-41.gh-issue-93442.4M4NDb.rst @@ -0,0 +1,3 @@ +Add C++ overloads for _Py_CAST_impl() to handle 0/NULL. This will allow C++ +extensions that pass 0 or NULL to macros using _Py_CAST() to continue to +compile. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-06-01-17-47-40.gh-issue-93418.24dJuc.rst b/Misc/NEWS.d/next/Core and Builtins/2022-06-01-17-47-40.gh-issue-93418.24dJuc.rst new file mode 100644 index 0000000000000..74ad06bfeee7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-06-01-17-47-40.gh-issue-93418.24dJuc.rst @@ -0,0 +1,2 @@ +Fixed an assert where an f-string has an equal sign '=' following an +expression, but there's no trailing brace. For example, f"{i=". diff --git a/Misc/NEWS.d/next/Library/2022-05-19-17-49-58.gh-issue-92932.o2peTh.rst b/Misc/NEWS.d/next/Library/2022-05-19-17-49-58.gh-issue-92932.o2peTh.rst new file mode 100644 index 0000000000000..cb76ac5cbd60e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-19-17-49-58.gh-issue-92932.o2peTh.rst @@ -0,0 +1,3 @@ +Now :func:`~dis.dis` and :func:`~dis.get_instructions` handle operand values +for instructions prefixed by ``EXTENDED_ARG_QUICK``. +Patch by Sam Gross and Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2022-05-22-16-08-01.gh-issue-89973.jc-Q4g.rst b/Misc/NEWS.d/next/Library/2022-05-22-16-08-01.gh-issue-89973.jc-Q4g.rst new file mode 100644 index 0000000000000..7e61fd7d46a0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-22-16-08-01.gh-issue-89973.jc-Q4g.rst @@ -0,0 +1,3 @@ +Fix :exc:`re.error` raised in :mod:`fnmatch` if the pattern contains a +character range with upper bound lower than lower bound (e.g. ``[c-a]``). +Now such ranges are interpreted as empty ranges. diff --git a/Misc/NEWS.d/next/Library/2022-05-26-23-10-55.gh-issue-93156.4XfDVN.rst b/Misc/NEWS.d/next/Library/2022-05-26-23-10-55.gh-issue-93156.4XfDVN.rst new file mode 100644 index 0000000000000..165baa08aaab1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-26-23-10-55.gh-issue-93156.4XfDVN.rst @@ -0,0 +1,2 @@ +Accessing the :attr:`pathlib.PurePath.parents` sequence of an absolute path +using negative index values produced incorrect results. diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst new file mode 100644 index 0000000000000..da81a1f6993db --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst @@ -0,0 +1,4 @@ +The deprecated mailcap module now refuses to inject unsafe text (filenames, +MIME types, parameters) into shell commands. Instead of using such text, it +will warn and act as if a match was not found (or for test commands, as if +the test failed). diff --git a/Misc/NEWS.d/next/Tests/2022-05-25-23-07-15.gh-issue-92886.Aki63_.rst b/Misc/NEWS.d/next/Tests/2022-05-25-23-07-15.gh-issue-92886.Aki63_.rst new file mode 100644 index 0000000000000..581f6bfea24b6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-05-25-23-07-15.gh-issue-92886.Aki63_.rst @@ -0,0 +1 @@ +Fixing tests that fail when running with optimizations (``-O``) in ``test_imaplib.py``. diff --git a/Misc/NEWS.d/next/Tests/2022-06-03-14-18-37.gh-issue-90473.7iXVRK.rst b/Misc/NEWS.d/next/Tests/2022-06-03-14-18-37.gh-issue-90473.7iXVRK.rst new file mode 100644 index 0000000000000..a3165a01111fb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-06-03-14-18-37.gh-issue-90473.7iXVRK.rst @@ -0,0 +1,2 @@ +Skip symlink tests on WASI. wasmtime uses ``openat2(2)`` with +``RESOLVE_BENEATH`` flag, which prevents symlinks with absolute paths. diff --git a/Misc/NEWS.d/next/Tests/2022-06-04-12-05-31.gh-issue-90473.RSpjF7.rst b/Misc/NEWS.d/next/Tests/2022-06-04-12-05-31.gh-issue-90473.RSpjF7.rst new file mode 100644 index 0000000000000..07d579995c091 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-06-04-12-05-31.gh-issue-90473.RSpjF7.rst @@ -0,0 +1 @@ +Skip tests on WASI that require symlinks with absolute paths. diff --git a/Misc/python.man b/Misc/python.man index c2e7e507e2fd6..f6c28647a4e99 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -69,10 +69,10 @@ python \- an interpreted, interactive, object-oriented programming language .B \-x ] [ -[ .B \-X .I option ] +[ .B \-? ] .br @@ -310,7 +310,8 @@ Set implementation specific option. The following options are available: more verbose than the default if the code is correct: new warnings are only emitted when an issue is detected. Effect of the developer mode: * Add default warning filter, as -W default - * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks() C function + * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks() + C function * Enable the faulthandler module to dump the Python traceback on a crash * Enable asyncio debug mode * Set the dev_mode attribute of sys.flags to True @@ -321,7 +322,19 @@ Set implementation specific option. The following options are available: otherwise activate automatically). See PYTHONUTF8 for more details -X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted at the - given directory instead of to the code tree. + given directory instead of to the code tree. + + -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None' + + -X no_debug_ranges: disable the inclusion of the tables mapping extra location + information (end line, start column offset and end column offset) to every + instruction in code objects. This is useful when smaller code objects and pyc + files are desired as well as suppressing the extra visual location indicators + when the interpreter displays tracebacks. + + -X frozen_modules=[on|off]: whether or not frozen modules should be used + The default is "on" (or "off" if you are running a local build). + .TP .B \-x Skip the first line of the source. This is intended for a DOS diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 22c0b147c1b89..2730030a15650 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -143,7 +143,7 @@ # needs -lncurses and -lpanel #@MODULE__CURSES_PANEL_TRUE@_curses_panel _curses_panel.c -@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c +@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/blob.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c # needs -lssl and -lcrypt @MODULE__SSL_TRUE@_ssl _ssl.c diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 19e9cd2d46f10..0caa92b2dc6e0 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -128,7 +128,7 @@ _bisect_insort_right_impl(PyObject *module, PyObject *a, PyObject *x, index = internal_bisect_right(a, x, lo, hi, key); } else { key_x = PyObject_CallOneArg(key, x); - if (x == NULL) { + if (key_x == NULL) { return NULL; } index = internal_bisect_right(a, key_x, lo, hi, key); @@ -256,7 +256,7 @@ _bisect_insort_left_impl(PyObject *module, PyObject *a, PyObject *x, index = internal_bisect_left(a, x, lo, hi, key); } else { key_x = PyObject_CallOneArg(key, x); - if (x == NULL) { + if (key_x == NULL) { return NULL; } index = internal_bisect_left(a, key_x, lo, hi, key); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index efb5278038f2f..e0bb4ee602c42 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5071,6 +5071,10 @@ datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, result_seconds = utc_to_seconds(year, month, day, hour, minute, second); + if (result_seconds == -1 && PyErr_Occurred()) { + return NULL; + } + /* Probe max_fold_seconds to detect a fold. */ probe_seconds = local(epoch + timet - max_fold_seconds); if (probe_seconds == -1) diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 1e27c5e0afb12..dd86fb5b64f3f 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -3,9 +3,9 @@ preserve [clinic start generated code]*/ static int -pysqlite_connection_init_impl(pysqlite_Connection *self, - const char *database, double timeout, - int detect_types, const char *isolation_level, +pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, + double timeout, int detect_types, + const char *isolation_level, int check_same_thread, PyObject *factory, int cache_size, int uri); @@ -19,7 +19,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; - const char *database = NULL; + PyObject *database; double timeout = 5.0; int detect_types = 0; const char *isolation_level = ""; @@ -32,9 +32,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) if (!fastargs) { goto exit; } - if (!clinic_fsconverter(fastargs[0], &database)) { - goto exit; - } + database = fastargs[0]; if (!noptargs) { goto skip_optional_pos; } @@ -102,9 +100,6 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) return_value = pysqlite_connection_init_impl((pysqlite_Connection *)self, database, timeout, detect_types, isolation_level, check_same_thread, factory, cache_size, uri); exit: - /* Cleanup for database */ - PyMem_Free((void *)database); - return return_value; } @@ -1236,4 +1231,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=d21767843c480a10 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb8908674e9f25ff input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/module.c.h b/Modules/_sqlite/clinic/module.c.h index 8f7008adef2b1..b088f4b8cfcd7 100644 --- a/Modules/_sqlite/clinic/module.c.h +++ b/Modules/_sqlite/clinic/module.c.h @@ -43,9 +43,7 @@ pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb if (!args) { goto exit; } - if (!PyUnicode_FSConverter(args[0], &database)) { - goto exit; - } + database = args[0]; if (!noptargs) { goto skip_optional_pos; } @@ -334,4 +332,4 @@ pysqlite_adapt(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d846459943008a9c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=10c4f942dc9f0c79 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 333847f0fafb9..7f7de8e709228 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -92,32 +92,6 @@ isolation_level_converter(PyObject *str_or_none, const char **result) return 1; } -static int -clinic_fsconverter(PyObject *pathlike, const char **result) -{ - PyObject *bytes = NULL; - Py_ssize_t len; - char *str; - - if (!PyUnicode_FSConverter(pathlike, &bytes)) { - goto error; - } - if (PyBytes_AsStringAndSize(bytes, &str, &len) < 0) { - goto error; - } - if ((*result = (const char *)PyMem_Malloc(len+1)) == NULL) { - goto error; - } - - memcpy((void *)(*result), str, len+1); - Py_DECREF(bytes); - return 1; - -error: - Py_XDECREF(bytes); - return 0; -} - #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/connection.c.h" #undef clinic_state @@ -159,25 +133,17 @@ new_statement_cache(pysqlite_Connection *self, pysqlite_state *state, } /*[python input] -class FSConverter_converter(CConverter): - type = "const char *" - converter = "clinic_fsconverter" - def converter_init(self): - self.c_default = "NULL" - def cleanup(self): - return f"PyMem_Free((void *){self.name});\n" - class IsolationLevel_converter(CConverter): type = "const char *" converter = "isolation_level_converter" [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=be142323885672ab]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=cbcfe85b253061c2]*/ /*[clinic input] _sqlite3.Connection.__init__ as pysqlite_connection_init - database: FSConverter + database: object timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" @@ -188,14 +154,19 @@ _sqlite3.Connection.__init__ as pysqlite_connection_init [clinic start generated code]*/ static int -pysqlite_connection_init_impl(pysqlite_Connection *self, - const char *database, double timeout, - int detect_types, const char *isolation_level, +pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, + double timeout, int detect_types, + const char *isolation_level, int check_same_thread, PyObject *factory, int cache_size, int uri) -/*[clinic end generated code: output=7d640ae1d83abfd4 input=342173993434ba1e]*/ +/*[clinic end generated code: output=839eb2fee4293bda input=b8ce63dc6f70a383]*/ { - if (PySys_Audit("sqlite3.connect", "s", database) < 0) { + if (PySys_Audit("sqlite3.connect", "O", database) < 0) { + return -1; + } + + PyObject *bytes; + if (!PyUnicode_FSConverter(database, &bytes)) { return -1; } @@ -210,7 +181,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, sqlite3 *db; int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_open_v2(database, &db, + rc = sqlite3_open_v2(PyBytes_AS_STRING(bytes), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | (uri ? SQLITE_OPEN_URI : 0), NULL); if (rc == SQLITE_OK) { @@ -218,6 +189,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, } Py_END_ALLOW_THREADS + Py_DECREF(bytes); if (db == NULL && rc == SQLITE_NOMEM) { PyErr_NoMemory(); return -1; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index fbc57c7cc739e..3f69427916ab8 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -46,7 +46,7 @@ module _sqlite3 /*[clinic input] _sqlite3.connect as pysqlite_connect - database: object(converter='PyUnicode_FSConverter') + database: object timeout: double = 5.0 detect_types: int = 0 isolation_level: object = NULL @@ -66,7 +66,7 @@ pysqlite_connect_impl(PyObject *module, PyObject *database, double timeout, int detect_types, PyObject *isolation_level, int check_same_thread, PyObject *factory, int cached_statements, int uri) -/*[clinic end generated code: output=450ac9078b4868bb input=ea6355ba55a78e12]*/ +/*[clinic end generated code: output=450ac9078b4868bb input=e16914663ddf93ce]*/ { if (isolation_level == NULL) { isolation_level = PyUnicode_FromString(""); @@ -81,7 +81,6 @@ pysqlite_connect_impl(PyObject *module, PyObject *database, double timeout, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri); - Py_DECREF(database); // needed bco. the AC FSConverter Py_DECREF(isolation_level); return res; } diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index bd9204da428af..491734f243849 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1323,6 +1323,7 @@ pattern_repr(PatternObject *obj) const char *name; int value; } flag_names[] = { + {"re.TEMPLATE", SRE_FLAG_TEMPLATE}, {"re.IGNORECASE", SRE_FLAG_IGNORECASE}, {"re.LOCALE", SRE_FLAG_LOCALE}, {"re.MULTILINE", SRE_FLAG_MULTILINE}, diff --git a/Modules/_sre/sre_constants.h b/Modules/_sre/sre_constants.h index d5de650b7025a..590d5be7cb4d9 100644 --- a/Modules/_sre/sre_constants.h +++ b/Modules/_sre/sre_constants.h @@ -85,6 +85,7 @@ #define SRE_CATEGORY_UNI_NOT_WORD 15 #define SRE_CATEGORY_UNI_LINEBREAK 16 #define SRE_CATEGORY_UNI_NOT_LINEBREAK 17 +#define SRE_FLAG_TEMPLATE 1 #define SRE_FLAG_IGNORECASE 2 #define SRE_FLAG_LOCALE 4 #define SRE_FLAG_MULTILINE 8 diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index be9ed50e0dbeb..a2d9ac807400b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1221,7 +1221,7 @@ test_type_from_ephemeral_spec(PyObject *self, PyObject *Py_UNUSED(ignored)) memcpy(name, NAME, sizeof(NAME)); doc = PyMem_New(char, sizeof(DOC)); - if (name == NULL) { + if (doc == NULL) { PyErr_NoMemory(); goto finally; } @@ -5955,6 +5955,51 @@ test_code_api(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +static int +record_func(PyObject *obj, PyFrameObject *f, int what, PyObject *arg) +{ + assert(PyList_Check(obj)); + PyObject *what_obj = NULL; + PyObject *line_obj = NULL; + PyObject *tuple = NULL; + int res = -1; + what_obj = PyLong_FromLong(what); + if (what_obj == NULL) { + goto error; + } + int line = PyFrame_GetLineNumber(f); + line_obj = PyLong_FromLong(line); + if (line_obj == NULL) { + goto error; + } + tuple = PyTuple_Pack(3, what_obj, line_obj, arg); + if (tuple == NULL) { + goto error; + } + PyTuple_SET_ITEM(tuple, 0, what_obj); + if (PyList_Append(obj, tuple)) { + goto error; + } + res = 0; +error: + Py_XDECREF(what_obj); + Py_XDECREF(line_obj); + Py_XDECREF(tuple); + return res; +} + +static PyObject * +settrace_to_record(PyObject *self, PyObject *list) +{ + + if (!PyList_Check(list)) { + PyErr_SetString(PyExc_TypeError, "argument must be a list"); + return NULL; + } + PyEval_SetTrace(record_func, list); + Py_RETURN_NONE; +} + static PyObject *negative_dictoffset(PyObject *, PyObject *); static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); @@ -6251,6 +6296,7 @@ static PyMethodDef TestMethods[] = { {"frame_getlasti", frame_getlasti, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, + {"settrace_to_record", settrace_to_record, METH_O, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 914b20b36f146..238de749fffc5 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -307,7 +307,7 @@ check_edit_cost(const char *a, const char *b, Py_ssize_t expected) goto exit; } b_obj = PyUnicode_FromString(b); - if (a_obj == NULL) { + if (b_obj == NULL) { goto exit; } Py_ssize_t result = _Py_UTF8_Edit_Cost(a_obj, b_obj, -1); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 9b1f186c5b6c7..d1df00111cf54 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1932,20 +1932,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, return -1; } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - // Switch to interpreter. - PyThreadState *new_tstate = PyInterpreterState_ThreadHead(interp); - PyThreadState *save1 = PyEval_SaveThread(); - - (void)PyThreadState_Swap(new_tstate); - - // Run the script. - _sharedexception *exc = NULL; - int result = _run_script(interp, codestr, shared, &exc); - - // Switch back. - PyEval_RestoreThread(save1); -#else // Switch to interpreter. PyThreadState *save_tstate = NULL; if (interp != PyInterpreterState_Get()) { @@ -1963,7 +1949,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, if (save_tstate != NULL) { PyThreadState_Swap(save_tstate); } -#endif // Propagate any exception out to the caller. if (exc != NULL) { diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 45dad8f0824df..3bda6e4bb8c02 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1195,14 +1195,6 @@ gc_collect_main(PyThreadState *tstate, int generation, assert(gcstate->garbage != NULL); assert(!_PyErr_Occurred(tstate)); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - if (tstate->interp->config._isolated_interpreter) { - // bpo-40533: The garbage collector must not be run on parallel on - // Python objects shared by multiple interpreters. - return 0; - } -#endif - if (gcstate->debug & DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); diff --git a/Modules/getpath.py b/Modules/getpath.py index 9aff19c0af7ed..47f075caf5551 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -228,6 +228,7 @@ def search_up(prefix, *landmarks, test=isfile): use_environment = config.get('use_environment', 1) pythonpath = config.get('module_search_paths') +pythonpath_was_set = config.get('module_search_paths_set') real_executable_dir = None stdlib_dir = None @@ -626,8 +627,8 @@ def search_up(prefix, *landmarks, test=isfile): config['module_search_paths'] = py_setpath.split(DELIM) config['module_search_paths_set'] = 1 -elif not pythonpath: - # If pythonpath was already set, we leave it alone. +elif not pythonpath_was_set: + # If pythonpath was already explicitly set or calculated, we leave it alone. # This won't matter in normal use, but if an embedded host is trying to # recalculate paths while running then we do not want to change it. pythonpath = [] diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 7475ef344b72b..18f9ddb909c02 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1481,7 +1481,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #elif defined(HAVE_CLOCK_GETTIME) && \ defined(CLOCK_PROCESS_CPUTIME_ID) && \ - !defined(__EMSCRIPTEN__) + !defined(__EMSCRIPTEN__) && !defined(__wasi__) #define HAVE_THREAD_TIME #if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 86861b7e28dce..47d308b8eb370 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -71,7 +71,7 @@ static PyStructSequence_Field floatinfo_fields[] = { {"min_exp", "DBL_MIN_EXP -- minimum int e such that radix**(e-1) " "is a normalized float"}, {"min_10_exp", "DBL_MIN_10_EXP -- minimum int e such that 10**e is " - "a normalized"}, + "a normalized float"}, {"dig", "DBL_DIG -- maximum number of decimal digits that " "can be faithfully represented in a float"}, {"mant_dig", "DBL_MANT_DIG -- mantissa digits"}, diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 5eeb1dbb338d2..59420816496f0 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -296,7 +296,7 @@ subs_tvars(PyObject *obj, PyObject *params, else { if (iparam >= 0) { if (iparam > varparam) { - iparam += nargs - nsubargs; + iparam += nargs - nparams; } arg = argitems[iparam]; } @@ -320,20 +320,85 @@ subs_tvars(PyObject *obj, PyObject *params, static int _is_unpacked_typevartuple(PyObject *arg) { - PyObject *meth; - int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth); + PyObject *tmp; + if (PyType_Check(arg)) { // TODO: Add test + return 0; + } + int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_is_unpacked_typevartuple__), &tmp); if (res > 0) { - PyObject *tmp = PyObject_CallNoArgs(meth); - Py_DECREF(meth); - if (tmp == NULL) { - return -1; - } res = PyObject_IsTrue(tmp); Py_DECREF(tmp); } return res; } +static PyObject * +_unpacked_tuple_args(PyObject *arg) +{ + PyObject *result; + assert(!PyType_Check(arg)); + // Fast path + if (_PyGenericAlias_Check(arg) && + ((gaobject *)arg)->starred && + ((gaobject *)arg)->origin == (PyObject *)&PyTuple_Type) + { + result = ((gaobject *)arg)->args; + Py_INCREF(result); + return result; + } + + if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked_tuple_args__), &result) > 0) { + if (result == Py_None) { + Py_DECREF(result); + return NULL; + } + return result; + } + return NULL; +} + +static PyObject * +_unpack_args(PyObject *item) +{ + PyObject *newargs = PyList_New(0); + if (newargs == NULL) { + return NULL; + } + int is_tuple = PyTuple_Check(item); + Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; + PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; + for (Py_ssize_t i = 0; i < nitems; i++) { + item = argitems[i]; + if (!PyType_Check(item)) { + PyObject *subargs = _unpacked_tuple_args(item); + if (subargs != NULL && + PyTuple_Check(subargs) && + !(PyTuple_GET_SIZE(subargs) && + PyTuple_GET_ITEM(subargs, PyTuple_GET_SIZE(subargs)-1) == Py_Ellipsis)) + { + if (PyList_SetSlice(newargs, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, subargs) < 0) { + Py_DECREF(subargs); + Py_DECREF(newargs); + return NULL; + } + Py_DECREF(subargs); + continue; + } + Py_XDECREF(subargs); + if (PyErr_Occurred()) { + Py_DECREF(newargs); + return NULL; + } + } + if (PyList_Append(newargs, item) < 0) { + Py_DECREF(newargs); + return NULL; + } + } + Py_SETREF(newargs, PySequence_Tuple(newargs)); + return newargs; +} + PyObject * _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { @@ -343,18 +408,26 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje "%R is not a generic class", self); } + item = _unpack_args(item); int is_tuple = PyTuple_Check(item); Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; - Py_ssize_t varparam = 0; - for (; varparam < nparams; varparam++) { - PyObject *param = PyTuple_GET_ITEM(parameters, varparam); + Py_ssize_t varparam = nparams; + for (Py_ssize_t i = 0; i < nparams; i++) { + PyObject *param = PyTuple_GET_ITEM(parameters, i); if (Py_TYPE(param)->tp_iter) { // TypeVarTuple - break; + if (varparam < nparams) { + Py_DECREF(item); + return PyErr_Format(PyExc_TypeError, + "More than one TypeVarTuple parameter in %S", + self); + } + varparam = i; } } if (varparam < nparams) { if (nitems < nparams - 1) { + Py_DECREF(item); return PyErr_Format(PyExc_TypeError, "Too few arguments for %R", self); @@ -362,10 +435,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } else { if (nitems != nparams) { + Py_DECREF(item); return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R", + "Too %s arguments for %R; actual %zd, expected %zd", nitems > nparams ? "many" : "few", - self); + self, nitems, nparams); } } /* Replace all type variables (specified by parameters) @@ -377,6 +451,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *newargs = PyTuple_New(nargs); if (newargs == NULL) { + Py_DECREF(item); return NULL; } for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { @@ -384,11 +459,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje int unpack = _is_unpacked_typevartuple(arg); if (unpack < 0) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } PyObject *subst; if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } if (subst) { @@ -397,6 +474,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (iparam == varparam) { Py_DECREF(subst); Py_DECREF(newargs); + Py_DECREF(item); PyErr_SetString(PyExc_TypeError, "Substitution of bare TypeVarTuple is not supported"); return NULL; @@ -412,6 +490,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } if (arg == NULL) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } if (unpack) { @@ -419,6 +498,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); Py_DECREF(arg); if (jarg < 0) { + Py_DECREF(item); return NULL; } } @@ -428,6 +508,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } } + Py_DECREF(item); return newargs; } @@ -454,6 +535,7 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); return res; @@ -518,6 +600,7 @@ static const char* const attr_exceptions[] = { "__args__", "__unpacked__", "__parameters__", + "__typing_unpacked_tuple_args__", "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", @@ -689,8 +772,20 @@ ga_parameters(PyObject *self, void *unused) return alias->parameters; } +static PyObject * +ga_unpacked_tuple_args(PyObject *self, void *unused) +{ + gaobject *alias = (gaobject *)self; + if (alias->starred && alias->origin == (PyObject *)&PyTuple_Type) { + Py_INCREF(alias->args); + return alias->args; + } + Py_RETURN_NONE; +} + static PyGetSetDef ga_properties[] = { {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, + {"__typing_unpacked_tuple_args__", ga_unpacked_tuple_args, (setter)NULL, NULL}, {0} }; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 45fe8985c2adb..8c26916882447 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -3156,7 +3156,7 @@ static PyMethodDef memory_methods[] = { /* Memoryview Iterator */ /**************************************************************************/ -static PyTypeObject PyMemoryIter_Type; +PyTypeObject _PyMemoryIter_Type; typedef struct { PyObject_HEAD @@ -3233,7 +3233,7 @@ memory_iter(PyObject *seq) } memoryiterobject *it; - it = PyObject_GC_New(memoryiterobject, &PyMemoryIter_Type); + it = PyObject_GC_New(memoryiterobject, &_PyMemoryIter_Type); if (it == NULL) { return NULL; } @@ -3246,7 +3246,7 @@ memory_iter(PyObject *seq) return (PyObject *)it; } -static PyTypeObject PyMemoryIter_Type = { +PyTypeObject _PyMemoryIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "memory_iterator", .tp_basicsize = sizeof(memoryiterobject), diff --git a/Objects/object.c b/Objects/object.c index d5f21b7c6aa19..303a22b6bfd01 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1845,6 +1845,7 @@ _PyTypes_InitState(PyInterpreterState *interp) extern PyTypeObject PyHKEY_Type; #endif extern PyTypeObject _Py_GenericAliasIterType; +extern PyTypeObject _PyMemoryIter_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -1944,6 +1945,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_Type, &_PyInterpreterID_Type, &_PyManagedBuffer_Type, + &_PyMemoryIter_Type, &_PyMethodWrapper_Type, &_PyNamespace_Type, &_PyNone_Type, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1bcfd9a9c52bc..c1bae0b9a6d62 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -54,11 +54,6 @@ typedef struct PySlot_Offset { } PySlot_Offset; -/* bpo-40521: Interned strings are shared by all subinterpreters */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -# define INTERN_NAME_STRINGS -#endif - static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -4009,7 +4004,7 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) if (name == NULL) return -1; } -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Interned strings are shared by all subinterpreters */ if (!PyUnicode_CHECK_INTERNED(name)) { PyUnicode_InternInPlace(&name); if (!PyUnicode_CHECK_INTERNED(name)) { @@ -4019,7 +4014,6 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) return -1; } } -#endif } else { /* Will fail in _PyObject_GenericSetAttrWithDict. */ @@ -8456,17 +8450,11 @@ _PyTypes_InitSlotDefs(void) for (slotdef *p = slotdefs; p->name; p++) { /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Interned strings are shared by all subinterpreters */ p->name_strobj = PyUnicode_InternFromString(p->name); if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) { return _PyStatus_NO_MEMORY(); } -#else - p->name_strobj = PyUnicode_FromString(p->name); - if (!p->name_strobj) { - return _PyStatus_NO_MEMORY(); - } -#endif } slotdefs_initialized = 1; return _PyStatus_OK(); @@ -8491,24 +8479,17 @@ update_slot(PyTypeObject *type, PyObject *name) int offset; assert(PyUnicode_CheckExact(name)); -#ifdef INTERN_NAME_STRINGS assert(PyUnicode_CHECK_INTERNED(name)); -#endif assert(slotdefs_initialized); pp = ptrs; for (p = slotdefs; p->name; p++) { assert(PyUnicode_CheckExact(p->name_strobj)); assert(PyUnicode_CheckExact(name)); -#ifdef INTERN_NAME_STRINGS + /* bpo-40521: Using interned strings. */ if (p->name_strobj == name) { *pp++ = p; } -#else - if (p->name_strobj == name || _PyUnicode_EQ(p->name_strobj, name)) { - *pp++ = p; - } -#endif } *pp = NULL; for (pp = ptrs; *pp; pp++) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 656c7ccc8e865..a7bd961c64a83 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -219,11 +219,6 @@ extern "C" { # define OVERALLOCATE_FACTOR 4 #endif -/* bpo-40521: Interned strings are shared by all interpreters. */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -# define INTERNED_STRINGS -#endif - /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -232,9 +227,7 @@ extern "C" { Another way to look at this is that to say that the actual reference count of a string is: s->ob_refcnt + (s->state ? 2 : 0) */ -#ifdef INTERNED_STRINGS static PyObject *interned = NULL; -#endif /* Forward declaration */ static inline int @@ -1924,10 +1917,8 @@ unicode_dealloc(PyObject *unicode) switch (PyUnicode_CHECK_INTERNED(unicode)) { case SSTATE_NOT_INTERNED: break; - case SSTATE_INTERNED_MORTAL: { -#ifdef INTERNED_STRINGS /* Revive the dead object temporarily. PyDict_DelItem() removes two references (key and value) which were ignored by PyUnicode_InternInPlace(). Use refcnt=3 rather than refcnt=2 @@ -1941,7 +1932,6 @@ unicode_dealloc(PyObject *unicode) } assert(Py_REFCNT(unicode) == 1); Py_SET_REFCNT(unicode, 0); -#endif break; } @@ -11314,13 +11304,11 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) if (PyUnicode_CHECK_INTERNED(left)) return 0; -#ifdef INTERNED_STRINGS assert(_PyUnicode_HASH(right_uni) != -1); Py_hash_t hash = _PyUnicode_HASH(left); if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) { return 0; } -#endif return unicode_compare_eq(left, right_uni); } @@ -15562,7 +15550,6 @@ PyUnicode_InternInPlace(PyObject **p) return; } -#ifdef INTERNED_STRINGS if (PyUnicode_READY(s) == -1) { PyErr_Clear(); return; @@ -15593,11 +15580,6 @@ PyUnicode_InternInPlace(PyObject **p) this. */ Py_SET_REFCNT(s, Py_REFCNT(s) - 2); _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; -#else - // PyDict expects that interned strings have their hash - // (PyASCIIObject.hash) already computed. - (void)unicode_hash(s); -#endif } void diff --git a/PC/launcher2.c b/PC/launcher2.c index 35c932aa329c8..ae11f4f024a90 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -386,6 +386,12 @@ typedef struct { int tagLength; // if true, treats 'tag' as a non-PEP 514 filter bool oldStyleTag; + // if true, ignores 'tag' when a high priority environment is found + // gh-92817: This is currently set when a tag is read from configuration or + // the environment, rather than the command line or a shebang line, and the + // only currently possible high priority environment is an active virtual + // environment + bool lowPriorityTag; // if true, we had an old-style tag with '-64' suffix, and so do not // want to match tags like '3.x-32' bool exclude32Bit; @@ -475,6 +481,7 @@ dumpSearchInfo(SearchInfo *search) DEBUG_2(company, companyLength); DEBUG_2(tag, tagLength); DEBUG_BOOL(oldStyleTag); + DEBUG_BOOL(lowPriorityTag); DEBUG_BOOL(exclude32Bit); DEBUG_BOOL(only32Bit); DEBUG_BOOL(allowDefaults); @@ -972,6 +979,9 @@ checkDefaults(SearchInfo *search) search->tagLength = n - (search->companyLength + 1); search->oldStyleTag = false; } + // gh-92817: allow a high priority env to be selected even if it + // doesn't match the tag + search->lowPriorityTag = true; } return 0; @@ -995,7 +1005,7 @@ typedef struct EnvironmentInfo { const wchar_t *executableArgs; const wchar_t *architecture; const wchar_t *displayName; - bool isActiveVenv; + bool highPriority; } EnvironmentInfo; @@ -1481,7 +1491,7 @@ virtualenvSearch(const SearchInfo *search, EnvironmentInfo **result) if (!env) { return RC_NO_MEMORY; } - env->isActiveVenv = true; + env->highPriority = true; env->internalSortKey = 20; exitCode = copyWstr(&env->displayName, L"Active venv"); if (exitCode) { @@ -1821,6 +1831,15 @@ _selectEnvironment(const SearchInfo *search, EnvironmentInfo *env, EnvironmentIn return 0; } + if (env->highPriority && search->lowPriorityTag) { + // This environment is marked high priority, and the search allows + // it to be selected even though a tag is specified, so select it + // gh-92817: this allows an active venv to be selected even when a + // default tag has been found in py.ini or the environment + *best = env; + return 0; + } + if (!search->oldStyleTag) { if (_companyMatches(search, env) && _tagMatches(search, env)) { // Because of how our sort tree is set up, we will walk up the diff --git a/PC/readme.txt b/PC/readme.txt index 0a96d269b0977..4e6dcf98c937f 100644 --- a/PC/readme.txt +++ b/PC/readme.txt @@ -18,7 +18,7 @@ All PC ports use this scheme to try to set up a module search path: 1) The script location; the current directory without script. 2) The PYTHONPATH variable, if set. - 3) For Win32 platforms (NT/95), paths specified in the Registry. + 3) Paths specified in the Registry. 4) Default directories lib, lib/win, lib/test, lib/tkinter; these are searched relative to the environment variable PYTHONHOME, if set, or relative to the executable and its @@ -26,8 +26,8 @@ All PC ports use this scheme to try to set up a module search path: or the current directory (not useful). 5) The directory containing the executable. -The best installation strategy is to put the Python executable (and -DLL, for Win32 platforms) in some convenient directory such as +The best installation strategy is to put the Python executable and +DLL in some convenient directory such as C:/python, and copy all library files and subdirectories (using XCOPY) to C:/python/lib. Then you don't need to set PYTHONPATH. Otherwise, set the environment variable PYTHONPATH to your Python search path. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 4241baadec4bd..d293445179885 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.2 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1n -set libraries=%libraries% sqlite-3.38.3.0 +set libraries=%libraries% sqlite-3.38.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.12.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.12.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 8d24393aa1861..df3efc631d154 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -18,6 +18,7 @@ true false false + false diff --git a/PCbuild/python.props b/PCbuild/python.props index e50fdc0aa61f3..7f10e7c45ef7b 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -61,7 +61,7 @@ $(EXTERNALS_DIR) $([System.IO.Path]::GetFullPath(`$(PySourcePath)externals`)) $(ExternalsDir)\ - $(ExternalsDir)sqlite-3.38.3.0\ + $(ExternalsDir)sqlite-3.38.4.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.2\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index da8947c938b51..e4cad75189c97 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -13,12 +13,10 @@ Quick Start Guide Building Python using Microsoft Visual C++ ------------------------------------------ -This directory is used to build CPython for Microsoft Windows NT version -6.0 or higher (Windows Vista, Windows Server 2008, or later) on 32 and 64 +This directory is used to build CPython for Microsoft Windows on 32- and 64- bit platforms. Using this directory requires an installation of -Microsoft Visual Studio 2017 (MSVC 14.1) with the *Python workload* and -its optional *Python native development* component selected. (For -command-line builds, Visual Studio 2015 may also be used.) +Microsoft Visual Studio (MSVC) with the *Python workload* and +its optional *Python native development* component selected. Building from the command line is recommended in order to obtain any external dependencies. To build, simply run the "build.bat" script without @@ -105,7 +103,7 @@ pythonw Prompt window pylauncher py.exe, the Python Launcher for Windows, see - http://docs.python.org/3/using/windows.html#launcher + https://docs.python.org/3/using/windows.html#launcher pywlauncher pyw.exe, a variant of py.exe that doesn't open a Command Prompt window @@ -168,14 +166,14 @@ _bz2 _lzma Python wrapper for version 5.2.2 of the liblzma compression library Homepage: - http://tukaani.org/xz/ + https://tukaani.org/xz/ _ssl Python wrapper for version 1.1.1k of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. Homepage: - http://www.openssl.org/ + https://www.openssl.org/ Building OpenSSL requires Perl on your path, and can be performed by running PCbuild\prepare_ssl.bat. This will retrieve the version of @@ -189,16 +187,16 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.38.3, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.38.4, which is itself built by sqlite3.vcxproj Homepage: - http://www.sqlite.org/ + https://www.sqlite.org/ _tkinter Wraps version 8.6.6 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. Homepage: - http://www.tcl.tk/ + https://www.tcl.tk/ Building Tcl and Tk can be performed by running PCbuild\prepare_tcltk.bat. This will retrieve the version of the @@ -257,7 +255,7 @@ It creates the PGI files, runs the unit test suite or PyBench with the PGI python, and finally creates the optimized files. See - http://msdn.microsoft.com/en-us/library/e7k32f4k(VS.140).aspx + https://docs.microsoft.com/en-us/cpp/build/profile-guided-optimizations for more on this topic. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3bfe320fe3b70..1101a3593dfe2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -488,6 +488,12 @@ def visitProduct(self, prod, name): class Obj2ModVisitor(PickleVisitor): + + attribute_special_defaults = { + "end_lineno": "lineno", + "end_col_offset": "col_offset", + } + @contextmanager def recursive_call(self, node, level): self.emit('if (_Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False) @@ -637,7 +643,13 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("if (tmp == NULL || tmp == Py_None) {", depth) self.emit("Py_CLEAR(tmp);", depth+1) if self.isNumeric(field): - self.emit("%s = 0;" % field.name, depth+1) + if field.name in self.attribute_special_defaults: + self.emit( + "%s = %s;" % (field.name, self.attribute_special_defaults[field.name]), + depth+1, + ) + else: + self.emit("%s = 0;" % field.name, depth+1) elif not self.isSimpleType(field): self.emit("%s = NULL;" % field.name, depth+1) else: diff --git a/Parser/parser.c b/Parser/parser.c index adc8d509eb7d7..08bf6d2945600 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -7,7 +7,11 @@ # define D(x) #endif -# define MAXSTACK 6000 +#ifdef __wasi__ +# define MAXSTACK 4000 +#else +# define MAXSTACK 6000 +#endif static const int n_keyword_lists = 9; static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 9c12d8ca101d0..5e94d477bc04a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -756,7 +756,9 @@ fstring_find_expr(Parser *p, const char **str, const char *end, int raw, int rec while (Py_ISSPACE(**str)) { *str += 1; } - + if (*str >= end) { + goto unexpected_end_of_string; + } /* Set *expr_text to the text of the expression. */ *expr_text = PyUnicode_FromStringAndSize(expr_start, *str-expr_start); if (!*expr_text) { diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 3861eaf978a38..e52a72d43bcbd 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5697,7 +5697,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -5714,7 +5714,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -8114,7 +8114,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -8131,7 +8131,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10291,7 +10291,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10308,7 +10308,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10755,7 +10755,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10772,7 +10772,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10877,7 +10877,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -10894,7 +10894,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; @@ -10999,7 +10999,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_lineno = 0; + end_lineno = lineno; } else { int res; @@ -11016,7 +11016,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - end_col_offset = 0; + end_col_offset = col_offset; } else { int res; diff --git a/Python/ast.c b/Python/ast.c index 607281e268553..a0321b58ba8cf 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -22,6 +22,27 @@ static int validate_stmt(struct validator *, stmt_ty); static int validate_expr(struct validator *, expr_ty, expr_context_ty); static int validate_pattern(struct validator *, pattern_ty, int); +#define VALIDATE_POSITIONS(node) \ + if (node->lineno > node->end_lineno) { \ + PyErr_Format(PyExc_ValueError, \ + "AST node line range (%d, %d) is not valid", \ + node->lineno, node->end_lineno); \ + return 0; \ + } \ + if ((node->lineno < 0 && node->end_lineno != node->lineno) || \ + (node->col_offset < 0 && node->col_offset != node->end_col_offset)) { \ + PyErr_Format(PyExc_ValueError, \ + "AST node column range (%d, %d) for line range (%d, %d) is not valid", \ + node->col_offset, node->end_col_offset, node->lineno, node->end_lineno); \ + return 0; \ + } \ + if (node->lineno == node->end_lineno && node->col_offset > node->end_col_offset) { \ + PyErr_Format(PyExc_ValueError, \ + "line %d, column %d-%d is not a valid range", \ + node->lineno, node->col_offset, node->end_col_offset); \ + return 0; \ + } + static int validate_name(PyObject *name) { @@ -75,6 +96,7 @@ validate_args(struct validator *state, asdl_arg_seq *args) Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = asdl_seq_GET(args, i); + VALIDATE_POSITIONS(arg); if (arg->annotation && !validate_expr(state, arg->annotation, Load)) return 0; } @@ -183,6 +205,7 @@ validate_constant(struct validator *state, PyObject *value) static int validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) { + VALIDATE_POSITIONS(exp); int ret = -1; if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, @@ -505,6 +528,7 @@ validate_capture(PyObject *name) static int validate_pattern(struct validator *state, pattern_ty p, int star_ok) { + VALIDATE_POSITIONS(p); int ret = -1; if (++state->recursion_depth > state->recursion_limit) { PyErr_SetString(PyExc_RecursionError, @@ -674,6 +698,7 @@ validate_body(struct validator *state, asdl_stmt_seq *body, const char *owner) static int validate_stmt(struct validator *state, stmt_ty stmt) { + VALIDATE_POSITIONS(stmt); int ret = -1; Py_ssize_t i; if (++state->recursion_depth > state->recursion_limit) { @@ -807,6 +832,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) } for (i = 0; i < asdl_seq_LEN(stmt->v.Try.handlers); i++) { excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i); + VALIDATE_POSITIONS(handler); if ((handler->v.ExceptHandler.type && !validate_expr(state, handler->v.ExceptHandler.type, Load)) || !validate_body(state, handler->v.ExceptHandler.body, "ExceptHandler")) diff --git a/Python/ceval.c b/Python/ceval.c index b2735a1b2d395..b8b27a815fb25 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -103,7 +103,6 @@ static PyObject * do_call_core( PyObject *callargs, PyObject *kwdict, int use_tracing); #ifdef LLTRACE -static int lltrace; static void dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) { @@ -349,21 +348,6 @@ _Py_FatalError_TstateNULL(const char *func) "(the current Python thread state is NULL)"); } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -int -_PyEval_ThreadsInitialized(PyInterpreterState *interp) -{ - return gil_created(&interp->ceval.gil); -} - -int -PyEval_ThreadsInitialized(void) -{ - // Fatal error if there is no current interpreter - PyInterpreterState *interp = PyInterpreterState_Get(); - return _PyEval_ThreadsInitialized(interp); -} -#else int _PyEval_ThreadsInitialized(_PyRuntimeState *runtime) { @@ -376,25 +360,18 @@ PyEval_ThreadsInitialized(void) _PyRuntimeState *runtime = &_PyRuntime; return _PyEval_ThreadsInitialized(runtime); } -#endif PyStatus _PyEval_InitGIL(PyThreadState *tstate) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return _PyStatus_OK(); } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; -#else struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; -#endif assert(!gil_created(gil)); PyThread_init_thread(); @@ -409,20 +386,14 @@ _PyEval_InitGIL(PyThreadState *tstate) void _PyEval_FiniGIL(PyInterpreterState *interp) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ return; } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; -#endif if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -486,13 +457,9 @@ PyEval_AcquireThread(PyThreadState *tstate) take_gil(tstate); struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - (void)_PyThreadState_Swap(gilstate, tstate); -#else if (_PyThreadState_Swap(gilstate, tstate) != NULL) { Py_FatalError("non-NULL old thread state"); } -#endif } void @@ -519,11 +486,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate) { _PyRuntimeState *runtime = tstate->interp->runtime; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &tstate->interp->ceval.gil; -#else struct _gil_runtime_state *gil = &runtime->ceval.gil; -#endif if (!gil_created(gil)) { return _PyStatus_OK(); } @@ -555,21 +518,12 @@ PyThreadState * PyEval_SaveThread(void) { _PyRuntimeState *runtime = &_PyRuntime; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThreadState *old_tstate = _PyThreadState_GET(); - PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, old_tstate); -#else PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL); -#endif _Py_EnsureTstateNotNULL(tstate); struct _ceval_runtime_state *ceval = &runtime->ceval; struct _ceval_state *ceval2 = &tstate->interp->ceval; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - assert(gil_created(&ceval2->gil)); -#else assert(gil_created(&ceval->gil)); -#endif drop_gil(ceval, ceval2, tstate); return tstate; } @@ -833,9 +787,7 @@ Py_MakePendingCalls(void) void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS _gil_initialize(&ceval->gil); -#endif } void @@ -845,10 +797,6 @@ _PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock) assert(pending->lock == NULL); pending->lock = pending_lock; - -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - _gil_initialize(&ceval->gil); -#endif } void @@ -1263,13 +1211,9 @@ eval_frame_handle_pending(PyThreadState *tstate) take_gil(tstate); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - (void)_PyThreadState_Swap(&runtime->gilstate, tstate); -#else if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) { Py_FatalError("orphan tstate"); } -#endif } /* Check for asynchronous exception. */ @@ -1715,6 +1659,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; +#ifdef LLTRACE + int lltrace = 0; +#endif _PyCFrame cframe; CallShape call_shape; @@ -4537,7 +4484,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_METHOD) { PREDICTED(LOAD_METHOD); - /* Designed to work in tandem with CALL_METHOD. */ + /* Designed to work in tandem with PRECALL. */ PyObject *name = GETITEM(names, oparg); PyObject *obj = TOP(); PyObject *meth = NULL; @@ -4562,7 +4509,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* meth is not an unbound method (but a regular attr, or something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal - CALL_METHOD that it's not a method call. + PRECALL that it's not a method call. NULL | meth | arg1 | ... | argN */ @@ -5680,6 +5627,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TRACE_FUNCTION_ENTRY(); DTRACE_FUNCTION_ENTRY(); break; + case POP_TOP: + if (_Py_OPCODE(next_instr[-1]) == RETURN_GENERATOR) { + /* Frame not fully initialized */ + break; + } + /* fall through */ default: /* line-by-line tracing support */ if (PyDTrace_LINE_ENABLED()) { diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index 9b8b43253f04d..1b2dc7f8e1dc3 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -144,11 +144,7 @@ static void drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, PyThreadState *tstate) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &ceval2->gil; -#else struct _gil_runtime_state *gil = &ceval->gil; -#endif if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -232,11 +228,7 @@ take_gil(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; struct _ceval_runtime_state *ceval = &interp->runtime->ceval; struct _ceval_state *ceval2 = &interp->ceval; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state *gil = &ceval2->gil; -#else struct _gil_runtime_state *gil = &ceval->gil; -#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -328,22 +320,12 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyInterpreterState *interp = PyInterpreterState_Get(); - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; -#endif gil->interval = microseconds; } unsigned long _PyEval_GetSwitchInterval() { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyInterpreterState *interp = PyInterpreterState_Get(); - struct _gil_runtime_state *gil = &interp->ceval.gil; -#else struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; -#endif return gil->interval; } diff --git a/Python/compile.c b/Python/compile.c index 10d6307a48406..cc0d76e0384c4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3152,6 +3152,8 @@ compiler_async_for(struct compiler *c, stmt_ty s) /* Success block for __anext__ */ VISIT(c, expr, s->v.AsyncFor.target); VISIT_SEQ(c, stmt, s->v.AsyncFor.body); + /* Mark jump as artificial */ + UNSET_LOC(c); ADDOP_JUMP(c, JUMP, start); compiler_pop_fblock(c, FOR_LOOP, start); @@ -9280,7 +9282,15 @@ trim_unused_consts(struct compiler *c, struct assembler *a, PyObject *consts) static inline int is_exit_without_lineno(basicblock *b) { - return b->b_exit && b->b_instr[0].i_lineno < 0; + if (!b->b_exit) { + return 0; + } + for (int i = 0; i < b->b_iused; i++) { + if (b->b_instr[i].i_lineno >= 0) { + return 0; + } + } + return 1; } /* PEP 626 mandates that the f_lineno of a frame is correct diff --git a/Python/hamt.c b/Python/hamt.c index c3cb4e6fda450..908c253187031 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -409,14 +409,22 @@ hamt_hash(PyObject *o) return -1; } - /* While it's suboptimal to reduce Python's 64 bit hash to + /* While it's somewhat suboptimal to reduce Python's 64 bit hash to 32 bits via XOR, it seems that the resulting hash function is good enough (this is also how Long type is hashed in Java.) Storing 10, 100, 1000 Python strings results in a relatively shallow and uniform tree structure. - Please don't change this hashing algorithm, as there are many - tests that test some exact tree shape to cover all code paths. + Also it's worth noting that it would be possible to adapt the tree + structure to 64 bit hashes, but that would increase memory pressure + and provide little to no performance benefits for collections with + fewer than billions of key/value pairs. + + Important: do not change this hash reducing function. There are many + tests that need an exact tree shape to cover all code paths and + we do that by specifying concrete values for test data's `__hash__`. + If this function is changed most of the regression tests would + become useless. */ int32_t xored = (int32_t)(hash & 0xffffffffl) ^ (int32_t)(hash >> 32); return xored == -1 ? -2 : xored; diff --git a/Python/initconfig.c b/Python/initconfig.c index a623973f95373..5c9c7ee41e753 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -38,7 +38,7 @@ Options and arguments (and corresponding environment variables):\n\ -d : turn on parser debugging output (for experts only, only works on\n\ debug builds); also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ --h : print this help message and exit (also --help)\n\ +-h : print this help message and exit (also -? or --help)\n\ "; static const char usage_2[] = "\ -i : inspect interactively after running script; forces a prompt even\n\ @@ -65,7 +65,6 @@ static const char usage_3[] = "\ also PYTHONWARNINGS=arg\n\ -x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ -X opt : set implementation-specific option. The following options are available:\n\ -\n\ -X faulthandler: enable faulthandler\n\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in the\n\ @@ -82,7 +81,8 @@ static const char usage_3[] = "\ checks which are too expensive to be enabled by default. Effect of the\n\ developer mode:\n\ * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks() C function\n\ + * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks()\n\ + C function\n\ * Enable the faulthandler module to dump the Python traceback on a crash\n\ * Enable asyncio debug mode\n\ * Set the dev_mode attribute of sys.flags to True\n\ @@ -100,7 +100,6 @@ static const char usage_3[] = "\ when the interpreter displays tracebacks.\n\ -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ The default is \"on\" (or \"off\" if you are running a local build).\n\ -\n\ --check-hash-based-pycs always|default|never:\n\ control how Python invalidates hash-based .pyc files\n\ "; @@ -141,7 +140,7 @@ static const char usage_6[] = "PYTHONNODEBUGRANGES: If this variable is set, it disables the inclusion of the \n" " tables mapping extra location information (end line, start column offset \n" " and end column offset) to every instruction in code objects. This is useful \n" -" when smaller cothe de objects and pyc files are desired as well as suppressing the \n" +" when smaller code objects and pyc files are desired as well as suppressing the \n" " extra visual location indicators when the interpreter displays tracebacks.\n"; #if defined(MS_WINDOWS) diff --git a/Python/preconfig.c b/Python/preconfig.c index afa16cccf32e9..0deb07a893ddf 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -294,17 +294,7 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) config->coerce_c_locale_warn = 0; config->dev_mode = -1; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - /* bpo-40512: pymalloc is not compatible with subinterpreters, - force usage of libc malloc() which is thread-safe. */ -#ifdef Py_DEBUG - config->allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; -#else - config->allocator = PYMEM_ALLOCATOR_MALLOC; -#endif -#else config->allocator = PYMEM_ALLOCATOR_NOT_SET; -#endif #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = -1; #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 273f6d62b2a20..960a38aebef8d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1986,12 +1986,10 @@ new_interpreter(PyThreadState **tstate_p, int isolated_subinterpreter) /* Copy the current interpreter config into the new interpreter */ const PyConfig *config; -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS if (save_tstate != NULL) { config = _PyInterpreterState_GetConfig(save_tstate->interp); } else -#endif { /* No current thread state, copy from the main interpreter */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); diff --git a/Python/pystate.c b/Python/pystate.c index 3e28a6ab69a98..df56c0530f05b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1165,14 +1165,6 @@ _PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) } -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -PyThreadState* -_PyThreadState_GetTSS(void) { - return PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey); -} -#endif - - PyThreadState * _PyThreadState_UncheckedGet(void) { @@ -1192,11 +1184,7 @@ PyThreadState_Get(void) PyThreadState * _PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThreadState *oldts = _PyThreadState_GetTSS(); -#else PyThreadState *oldts = _PyRuntimeGILState_GetThreadState(gilstate); -#endif _PyRuntimeGILState_SetThreadState(gilstate, newts); /* It should not be possible for more than one thread state @@ -1214,9 +1202,6 @@ _PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *new Py_FatalError("Invalid thread state for this thread"); errno = err; } -#endif -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - PyThread_tss_set(&gilstate->autoTSSkey, newts); #endif return oldts; } @@ -1665,9 +1650,7 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS assert(_PyEval_ThreadsInitialized(runtime)); -#endif assert(gilstate->autoInterpreterState); PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index edd1d1f23fdea..4f8b4cc17f2c1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3302,7 +3302,10 @@ PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) void PySys_SetArgv(int argc, wchar_t **argv) { +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS PySys_SetArgvEx(argc, argv, Py_IsolatedFlag == 0); +_Py_COMP_DIAG_POP } /* Reimplementation of PyFile_WriteString() no calling indirectly diff --git a/README.rst b/README.rst index 5a2b1ad1f8b44..802f9528cb8ab 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.11.0 beta 1 +This is Python version 3.11.0 beta 3 ==================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index bf0fe5bed5a76..5ad4f879a33f7 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1628,10 +1628,16 @@ def parse_clinic_block(self, dsl_name): def is_stop_line(line): # make sure to recognize stop line even if it # doesn't end with EOL (it could be the very end of the file) - if not line.startswith(stop_line): + if line.startswith(stop_line): + remainder = line[len(stop_line):] + if remainder and not remainder.isspace(): + fail(f"Garbage after stop line: {remainder!r}") + return True + else: + # gh-92256: don't allow incorrectly formatted stop lines + if line.lstrip().startswith(stop_line): + fail(f"Whitespace is not allowed before the stop line: {line!r}") return False - remainder = line[len(stop_line):] - return (not remainder) or remainder.isspace() # consume body of program while self.input: diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 56a1e5a5a14fb..65bfd5900a696 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -37,7 +37,11 @@ # define D(x) #endif -# define MAXSTACK 6000 +#ifdef __wasi__ +# define MAXSTACK 4000 +#else +# define MAXSTACK 6000 +#endif """ diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 83806f0581ace..6bb9bd9978402 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -220,10 +220,44 @@ AddType application/wasm wasm # WASI (wasm32-wasi) -WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) and -currently [wasix](https://github.com/singlestore-labs/wasix) for POSIX +WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 15.0+ +and currently [wasix](https://github.com/singlestore-labs/wasix) for POSIX compatibility stubs. +## WASI limitations and issues (WASI SDK 15.0) + +A lot of Emscripten limitations also apply to WASI. Noticable restrictions +are: + +- Call stack size is limited. Default recursion limit and parser stack size + are smaller than in regular Python builds. +- ``socket(2)`` cannot create new socket file descriptors. WASI programs can + call read/write/accept on a file descriptor that is passed into the process. +- ``socket.gethostname()`` and host name resolution APIs like + ``socket.gethostbyname()`` are not implemented and always fail. +- ``chmod(2)`` is not available. It's not possible to modify file permissions, + yet. A future version of WASI may provide a limited ``set_permissions`` API. +- User/group related features like ``os.chown()``, ``os.getuid``, etc. are + stubs or fail with ``ENOTSUP``. +- File locking (``fcntl``) is not available. +- ``os.pipe()``, ``os.mkfifo()``, and ``os.mknod()`` are not supported. +- ``process_time`` does not work as expected because it's implemented using + wall clock. +- ``os.umask()`` is a stub. +- ``sys.executable`` is empty. +- ``/dev/null`` / ``os.devnull`` may not be available. +- ``os.utime*()`` is buggy in WASM SDK 15.0, see + [utimensat() with timespec=NULL sets wrong time](https://github.com/bytecodealliance/wasmtime/issues/4184) +- ``os.symlink()`` fails with ``PermissionError`` when attempting to create a + symlink with an absolute path with wasmtime 0.36.0. The wasmtime runtime + uses ``openat2(2)`` syscall with flag ``RESOLVE_BENEATH`` to open files. + The flag causes the syscall to reject symlinks with absolute paths. +- ``os.curdir`` (aka ``.``) seems to behave differently, which breaks some + ``importlib`` tests that add ``.`` to ``sys.path`` and indirectly + ``sys.path_importer_cache``. +- WASI runtime environments may not provide a dedicated temp directory. + + # Detect WebAssembly builds ## Python code diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 255e99c279a0a..a6fcbed48fa81 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -17,3 +17,25 @@ ac_cv_header_sys_resource_h=no # undefined symbols / unsupported features ac_cv_func_eventfd=no + +# WASI SDK 15.0 has no pipe syscall. +ac_cv_func_pipe=no + +# WASI SDK 15.0 cannot create fifos and special files. +ac_cv_func_mkfifo=no +ac_cv_func_mkfifoat=no +ac_cv_func_mknod=no +ac_cv_func_mknodat=no +ac_cv_func_makedev=no + +# fdopendir() fails on SDK 15.0, +# OSError: [Errno 28] Invalid argument: '.' +ac_cv_func_fdopendir=no + +# WASIX stubs we don't want to use. +ac_cv_func_kill=no + +# WASI sockets are limited to operations on given socket fd and inet sockets. +# Disable AF_UNIX and AF_PACKET support, see socketmodule.h. +ac_cv_header_sys_un_h=no +ac_cv_header_netpacket_packet_h=no diff --git a/configure b/configure index b57c6f3a45abe..e91d04b3b6621 100755 --- a/configure +++ b/configure @@ -654,10 +654,10 @@ MODULE_BINASCII_FALSE MODULE_BINASCII_TRUE MODULE_ZLIB_FALSE MODULE_ZLIB_TRUE -MODULE__TKINTER_FALSE -MODULE__TKINTER_TRUE MODULE__UUID_FALSE MODULE__UUID_TRUE +MODULE__TKINTER_FALSE +MODULE__TKINTER_TRUE MODULE__SQLITE3_FALSE MODULE__SQLITE3_TRUE MODULE_NIS_FALSE @@ -1061,7 +1061,6 @@ with_openssl with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes -with_experimental_isolated_subinterpreters enable_test_modules ' ac_precious_vars='build_alias @@ -1861,9 +1860,6 @@ Optional Packages: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 builtin hash modules, md5, sha1, sha256, sha512, sha3 (with shake), blake2 - --with-experimental-isolated-subinterpreters - better isolate subinterpreters, experimental build - mode (default is no) Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -7922,6 +7918,8 @@ $as_echo "#define _WASI_EMULATED_PROCESS_CLOCKS 1" >>confdefs.h LIBS="$LIBS -lwasi-emulated-signal -lwasi-emulated-getpid -lwasi-emulated-process-clocks" echo "#define _WASI_EMULATED_SIGNAL 1" >> confdefs.h + as_fn_append LDFLAGS_NODIST " -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760" + ;; #( *) : ;; @@ -14231,19 +14229,26 @@ int domain = AF_INET6; _ACEOF if ac_fn_c_try_compile "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } ipv6=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } ipv6=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +case $ac_sys_system in #( + WASI) : + ipv6=no + ;; #( + *) : + ;; +esac + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ipv6" >&5 +$as_echo "$ipv6" >&6; } + if test "$ipv6" = "yes"; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking if RFC2553 API is available" >&5 $as_echo_n "checking if RFC2553 API is available... " >&6; } @@ -14557,6 +14562,8 @@ then case $ac_sys_system in #( Emscripten) : with_pymalloc="no" ;; #( + WASI) : + with_pymalloc="no" ;; #( *) : with_pymalloc="yes" ;; @@ -22464,30 +22471,6 @@ fi fi -# --with-experimental-isolated-subinterpreters - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-experimental-isolated-subinterpreters" >&5 -$as_echo_n "checking for --with-experimental-isolated-subinterpreters... " >&6; } - -# Check whether --with-experimental-isolated-subinterpreters was given. -if test "${with_experimental_isolated_subinterpreters+set}" = set; then : - withval=$with_experimental_isolated_subinterpreters; -if test "$withval" != no -then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; }; - $as_echo "#define EXPERIMENTAL_ISOLATED_SUBINTERPRETERS 1" >>confdefs.h - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; }; -fi -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --disable-test-modules" >&5 @@ -22597,6 +22580,16 @@ case $ac_sys_system in #( ;; #( WASI/*) : + + + py_cv_module__ctypes_test=n/a + py_cv_module_fcntl=n/a + py_cv_module_mmap=n/a + py_cv_module_resource=n/a + py_cv_module_termios=n/a + py_cv_module_=n/a + + ;; #( *) : ;; @@ -24197,40 +24190,6 @@ fi $as_echo "$py_cv_module__sqlite3" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _uuid" >&5 -$as_echo_n "checking for stdlib extension module _uuid... " >&6; } - if test "$py_cv_module__uuid" != "n/a"; then : - - if true; then : - if test "$have_uuid" = "yes"; then : - py_cv_module__uuid=yes -else - py_cv_module__uuid=missing -fi -else - py_cv_module__uuid=disabled -fi - -fi - as_fn_append MODULE_BLOCK "MODULE__UUID=$py_cv_module__uuid$as_nl" - if test "x$py_cv_module__uuid" = xyes; then : - - as_fn_append MODULE_BLOCK "MODULE__UUID_CFLAGS=$LIBUUID_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__UUID_LDFLAGS=$LIBUUID_LIBS$as_nl" - -fi - if test "$py_cv_module__uuid" = yes; then - MODULE__UUID_TRUE= - MODULE__UUID_FALSE='#' -else - MODULE__UUID_TRUE='#' - MODULE__UUID_FALSE= -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__uuid" >&5 -$as_echo "$py_cv_module__uuid" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _tkinter" >&5 $as_echo_n "checking for stdlib extension module _tkinter... " >&6; } if test "$py_cv_module__tkinter" != "n/a"; then : @@ -24265,6 +24224,40 @@ fi $as_echo "$py_cv_module__tkinter" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _uuid" >&5 +$as_echo_n "checking for stdlib extension module _uuid... " >&6; } + if test "$py_cv_module__uuid" != "n/a"; then : + + if true; then : + if test "$have_uuid" = "yes"; then : + py_cv_module__uuid=yes +else + py_cv_module__uuid=missing +fi +else + py_cv_module__uuid=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__UUID=$py_cv_module__uuid$as_nl" + if test "x$py_cv_module__uuid" = xyes; then : + + as_fn_append MODULE_BLOCK "MODULE__UUID_CFLAGS=$LIBUUID_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__UUID_LDFLAGS=$LIBUUID_LIBS$as_nl" + +fi + if test "$py_cv_module__uuid" = yes; then + MODULE__UUID_TRUE= + MODULE__UUID_FALSE='#' +else + MODULE__UUID_TRUE='#' + MODULE__UUID_FALSE= +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__uuid" >&5 +$as_echo "$py_cv_module__uuid" >&6; } + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module zlib" >&5 $as_echo_n "checking for stdlib extension module zlib... " >&6; } @@ -25121,14 +25114,14 @@ if test -z "${MODULE__SQLITE3_TRUE}" && test -z "${MODULE__SQLITE3_FALSE}"; then as_fn_error $? "conditional \"MODULE__SQLITE3\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__UUID_TRUE}" && test -z "${MODULE__UUID_FALSE}"; then - as_fn_error $? "conditional \"MODULE__UUID\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${MODULE__TKINTER_TRUE}" && test -z "${MODULE__TKINTER_FALSE}"; then as_fn_error $? "conditional \"MODULE__TKINTER\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__UUID_TRUE}" && test -z "${MODULE__UUID_FALSE}"; then + as_fn_error $? "conditional \"MODULE__UUID\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_ZLIB_TRUE}" && test -z "${MODULE_ZLIB_FALSE}"; then as_fn_error $? "conditional \"MODULE_ZLIB\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 07b8885f1e482..2e2e24ceca93b 100644 --- a/configure.ac +++ b/configure.ac @@ -1994,6 +1994,10 @@ AS_CASE([$ac_sys_system], AC_DEFINE([_WASI_EMULATED_PROCESS_CLOCKS], [1], [Define to 1 if you want to emulate process clocks on WASI]) LIBS="$LIBS -lwasi-emulated-signal -lwasi-emulated-getpid -lwasi-emulated-process-clocks" echo "#define _WASI_EMULATED_SIGNAL 1" >> confdefs.h + + dnl increase initial memory and stack size, move stack first + dnl https://github.com/WebAssembly/wasi-libc/issues/233 + AS_VAR_APPEND([LDFLAGS_NODIST], [" -z stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760"]) ] ) @@ -4046,13 +4050,17 @@ dnl the check does not work on cross compilation case... #include #include ]], [[int domain = AF_INET6;]])],[ - AC_MSG_RESULT(yes) ipv6=yes ],[ - AC_MSG_RESULT(no) ipv6=no ]) +AS_CASE([$ac_sys_system], + [WASI], [ipv6=no] +) + +AC_MSG_RESULT([$ipv6]) + if test "$ipv6" = "yes"; then AC_MSG_CHECKING(if RFC2553 API is available) AC_COMPILE_IFELSE([ @@ -4232,9 +4240,10 @@ AC_ARG_WITH(pymalloc, if test -z "$with_pymalloc" then - dnl default to yes except for wasm32-emscripten + dnl default to yes except for wasm32-emscripten and wasm32-wasi. AS_CASE([$ac_sys_system], [Emscripten], [with_pymalloc="no"], + [WASI], [with_pymalloc="no"], [with_pymalloc="yes"] ) fi @@ -6598,23 +6607,6 @@ AS_VAR_IF([with_builtin_blake2], [yes], [ ], [have_libb2=no]) ]) -# --with-experimental-isolated-subinterpreters -AH_TEMPLATE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS, - [Better isolate subinterpreters, experimental build mode.]) -AC_MSG_CHECKING(for --with-experimental-isolated-subinterpreters) -AC_ARG_WITH(experimental-isolated-subinterpreters, - AS_HELP_STRING([--with-experimental-isolated-subinterpreters], - [better isolate subinterpreters, experimental build mode (default is no)]), -[ -if test "$withval" != no -then - AC_MSG_RESULT(yes); - AC_DEFINE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS) -else - AC_MSG_RESULT(no); -fi], -[AC_MSG_RESULT(no)]) - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. AC_MSG_CHECKING([for --disable-test-modules]) @@ -6680,7 +6672,16 @@ AS_CASE([$ac_sys_system], ) ], [Emscripten/node*], [], - [WASI/*], [] + [WASI/*], [ + dnl WASI SDK 15.0 does not support file locking, mmap, and more. + PY_STDLIB_MOD_SET_NA( + [_ctypes_test], + [fcntl], + [mmap], + [resource], + [termios], + ) + ] ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -6869,12 +6870,12 @@ PY_STDLIB_MOD([_sqlite3], [test "$have_sqlite3" = "yes"], [test "$have_supported_sqlite3" = "yes"], [$LIBSQLITE3_CFLAGS], [$LIBSQLITE3_LIBS]) -dnl PY_STDLIB_MOD([_tkinter], [], [], [], []) +PY_STDLIB_MOD([_tkinter], + [], [test "$have_tcltk" = "yes"], + [$TCLTK_CFLAGS], [$TCLTK_LIBS]) PY_STDLIB_MOD([_uuid], [], [test "$have_uuid" = "yes"], [$LIBUUID_CFLAGS], [$LIBUUID_LIBS]) -PY_STDLIB_MOD([_tkinter], [], - [test "$have_tcltk" = "yes"], [$TCLTK_CFLAGS], [$TCLTK_LIBS]) dnl compression libs PY_STDLIB_MOD([zlib], [], [test "$have_zlib" = yes], diff --git a/pyconfig.h.in b/pyconfig.h.in index 383fd47dd43c8..b325ef259c8c0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -44,9 +44,6 @@ /* Define if --enable-ipv6 is specified */ #undef ENABLE_IPV6 -/* Better isolate subinterpreters, experimental build mode. */ -#undef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - /* Define to 1 if your system stores words within floats with the most significant word first */ #undef FLOAT_WORDS_BIGENDIAN diff --git a/setup.py b/setup.py index f45cd6de33749..4c497346e8d7e 100644 --- a/setup.py +++ b/setup.py @@ -1408,9 +1408,6 @@ def detect_ctypes(self): # finding some -z option for the Sun compiler. extra_link_args.append('-mimpure-text') - elif HOST_PLATFORM.startswith('hp-ux'): - extra_link_args.append('-fPIC') - ext = Extension('_ctypes', include_dirs=include_dirs, extra_compile_args=extra_compile_args,