`."
+ ) from e
+ except RepositoryNotFoundError as e:
+ logger.error(e)
+ raise OSError(f"{path_or_repo} is not a local folder or a valid repository name on 'https://hf.co'.") from e
+ except RevisionNotFoundError as e:
+ logger.error(e)
+ raise OSError(
+ f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for this "
+ f"model name. Check the model page at 'https://huggingface.co/{path_or_repo}' for available revisions."
+ ) from e
+ except EntryNotFoundError:
+ return False # File does not exist
+ except requests.HTTPError:
+ # Any authentication/authorization error will be caught here => default to cache
+ return has_file_in_cache
+
+
+class PushToHubMixin:
+ """
+ A Mixin containing the functionality to push a model or tokenizer to the hub.
+ """
+
+ def _create_repo(
+ self,
+ repo_id: str,
+ private: Optional[bool] = None,
+ token: Optional[Union[bool, str]] = None,
+ repo_url: Optional[str] = None,
+ organization: Optional[str] = None,
+ ) -> str:
+ """
+ Create the repo if needed, cleans up repo_id with deprecated kwargs `repo_url` and `organization`, retrieves
+ the token.
+ """
+ if repo_url is not None:
+ warnings.warn(
+ "The `repo_url` argument is deprecated and will be removed in v5 of Transformers. Use `repo_id` "
+ "instead."
+ )
+ if repo_id is not None:
+ raise ValueError(
+ "`repo_id` and `repo_url` are both specified. Please set only the argument `repo_id`."
+ )
+ repo_id = repo_url.replace(f"{HUGGINGFACE_CO_RESOLVE_ENDPOINT}/", "")
+ if organization is not None:
+ warnings.warn(
+ "The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your "
+ "organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`)."
+ )
+ if not repo_id.startswith(organization):
+ if "/" in repo_id:
+ repo_id = repo_id.split("/")[-1]
+ repo_id = f"{organization}/{repo_id}"
+
+ url = create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True)
+ return url.repo_id
+
+ def _get_files_timestamps(self, working_dir: Union[str, os.PathLike]):
+ """
+ Returns the list of files with their last modification timestamp.
+ """
+ return {f: os.path.getmtime(os.path.join(working_dir, f)) for f in os.listdir(working_dir)}
+
+ def _upload_modified_files(
+ self,
+ working_dir: Union[str, os.PathLike],
+ repo_id: str,
+ files_timestamps: dict[str, float],
+ commit_message: Optional[str] = None,
+ token: Optional[Union[bool, str]] = None,
+ create_pr: bool = False,
+ revision: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ ):
+ """
+ Uploads all modified files in `working_dir` to `repo_id`, based on `files_timestamps`.
+ """
+ if commit_message is None:
+ if "Model" in self.__class__.__name__:
+ commit_message = "Upload model"
+ elif "Config" in self.__class__.__name__:
+ commit_message = "Upload config"
+ elif "Tokenizer" in self.__class__.__name__:
+ commit_message = "Upload tokenizer"
+ elif "FeatureExtractor" in self.__class__.__name__:
+ commit_message = "Upload feature extractor"
+ elif "Processor" in self.__class__.__name__:
+ commit_message = "Upload processor"
+ else:
+ commit_message = f"Upload {self.__class__.__name__}"
+ modified_files = [
+ f
+ for f in os.listdir(working_dir)
+ if f not in files_timestamps or os.path.getmtime(os.path.join(working_dir, f)) > files_timestamps[f]
+ ]
+
+ # filter for actual files + folders at the root level
+ modified_files = [
+ f
+ for f in modified_files
+ if os.path.isfile(os.path.join(working_dir, f)) or os.path.isdir(os.path.join(working_dir, f))
+ ]
+
+ operations = []
+ # upload standalone files
+ for file in modified_files:
+ if os.path.isdir(os.path.join(working_dir, file)):
+ # go over individual files of folder
+ for f in os.listdir(os.path.join(working_dir, file)):
+ operations.append(
+ CommitOperationAdd(
+ path_or_fileobj=os.path.join(working_dir, file, f), path_in_repo=os.path.join(file, f)
+ )
+ )
+ else:
+ operations.append(
+ CommitOperationAdd(path_or_fileobj=os.path.join(working_dir, file), path_in_repo=file)
+ )
+
+ if revision is not None and not revision.startswith("refs/pr"):
+ try:
+ create_branch(repo_id=repo_id, branch=revision, token=token, exist_ok=True)
+ except HfHubHTTPError as e:
+ if e.response.status_code == 403 and create_pr:
+ # If we are creating a PR on a repo we don't have access to, we can't create the branch.
+ # so let's assume the branch already exists. If it's not the case, an error will be raised when
+ # calling `create_commit` below.
+ pass
+ else:
+ raise
+
+ logger.info(f"Uploading the following files to {repo_id}: {','.join(modified_files)}")
+ return create_commit(
+ repo_id=repo_id,
+ operations=operations,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ token=token,
+ create_pr=create_pr,
+ revision=revision,
+ )
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ use_temp_dir: Optional[bool] = None,
+ commit_message: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[Union[bool, str]] = None,
+ max_shard_size: Optional[Union[int, str]] = "5GB",
+ create_pr: bool = False,
+ safe_serialization: bool = True,
+ revision: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ tags: Optional[list[str]] = None,
+ **deprecated_kwargs,
+ ) -> str:
+ """
+ Upload the {object_files} to the 🤗 Model Hub.
+
+ Parameters:
+ repo_id (`str`):
+ The name of the repository you want to push your {object} to. It should contain your organization name
+ when pushing to a given organization.
+ use_temp_dir (`bool`, *optional*):
+ Whether or not to use a temporary directory to store the files saved before they are pushed to the Hub.
+ Will default to `True` if there is no directory named like `repo_id`, `False` otherwise.
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload {object}"`.
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
+ token (`bool` or `str`, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`). Will default to `True` if `repo_url`
+ is not specified.
+ max_shard_size (`int` or `str`, *optional*, defaults to `"5GB"`):
+ Only applicable for models. The maximum size for a checkpoint before being sharded. Checkpoints shard
+ will then be each of size lower than this size. If expressed as a string, needs to be digits followed
+ by a unit (like `"5MB"`). We default it to `"5GB"` so that users can easily load models on free-tier
+ Google Colab instances without any CPU OOM issues.
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether or not to create a PR with the uploaded files or directly commit.
+ safe_serialization (`bool`, *optional*, defaults to `True`):
+ Whether or not to convert the model weights in safetensors format for safer serialization.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to.
+ commit_description (`str`, *optional*):
+ The description of the commit that will be created
+ tags (`list[str]`, *optional*):
+ List of tags to push on the Hub.
+
+ Examples:
+
+ ```python
+ from transformers import {object_class}
+
+ {object} = {object_class}.from_pretrained("google-bert/bert-base-cased")
+
+ # Push the {object} to your namespace with the name "my-finetuned-bert".
+ {object}.push_to_hub("my-finetuned-bert")
+
+ # Push the {object} to an organization with the name "my-finetuned-bert".
+ {object}.push_to_hub("huggingface/my-finetuned-bert")
+ ```
+ """
+ use_auth_token = deprecated_kwargs.pop("use_auth_token", None)
+ ignore_metadata_errors = deprecated_kwargs.pop("ignore_metadata_errors", False)
+ save_jinja_files = deprecated_kwargs.pop(
+ "save_jinja_files", None
+ ) # TODO: This is only used for testing and should be removed once save_jinja_files becomes the default
+ if use_auth_token is not None:
+ warnings.warn(
+ "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
+ FutureWarning,
+ )
+ if token is not None:
+ raise ValueError(
+ "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
+ )
+ token = use_auth_token
+
+ repo_path_or_name = deprecated_kwargs.pop("repo_path_or_name", None)
+ if repo_path_or_name is not None:
+ # Should use `repo_id` instead of `repo_path_or_name`. When using `repo_path_or_name`, we try to infer
+ # repo_id from the folder path, if it exists.
+ warnings.warn(
+ "The `repo_path_or_name` argument is deprecated and will be removed in v5 of Transformers. Use "
+ "`repo_id` instead.",
+ FutureWarning,
+ )
+ if repo_id is not None:
+ raise ValueError(
+ "`repo_id` and `repo_path_or_name` are both specified. Please set only the argument `repo_id`."
+ )
+ if os.path.isdir(repo_path_or_name):
+ # repo_path: infer repo_id from the path
+ repo_id = repo_path_or_name.split(os.path.sep)[-1]
+ working_dir = repo_id
+ else:
+ # repo_name: use it as repo_id
+ repo_id = repo_path_or_name
+ working_dir = repo_id.split("/")[-1]
+ else:
+ # Repo_id is passed correctly: infer working_dir from it
+ working_dir = repo_id.split("/")[-1]
+
+ # Deprecation warning will be sent after for repo_url and organization
+ repo_url = deprecated_kwargs.pop("repo_url", None)
+ organization = deprecated_kwargs.pop("organization", None)
+
+ repo_id = self._create_repo(
+ repo_id, private=private, token=token, repo_url=repo_url, organization=organization
+ )
+
+ # Create a new empty model card and eventually tag it
+ model_card = create_and_tag_model_card(
+ repo_id, tags, token=token, ignore_metadata_errors=ignore_metadata_errors
+ )
+
+ if use_temp_dir is None:
+ use_temp_dir = not os.path.isdir(working_dir)
+
+ with working_or_temp_dir(working_dir=working_dir, use_temp_dir=use_temp_dir) as work_dir:
+ files_timestamps = self._get_files_timestamps(work_dir)
+
+ # Save all files.
+ if save_jinja_files:
+ self.save_pretrained(
+ work_dir,
+ max_shard_size=max_shard_size,
+ safe_serialization=safe_serialization,
+ save_jinja_files=True,
+ )
+ else:
+ self.save_pretrained(work_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+
+ # Update model card if needed:
+ model_card.save(os.path.join(work_dir, "README.md"))
+
+ return self._upload_modified_files(
+ work_dir,
+ repo_id,
+ files_timestamps,
+ commit_message=commit_message,
+ token=token,
+ create_pr=create_pr,
+ revision=revision,
+ commit_description=commit_description,
+ )
+
+
+def send_example_telemetry(example_name, *example_args, framework="pytorch"):
+ """
+ Sends telemetry that helps tracking the examples use.
+
+ Args:
+ example_name (`str`): The name of the example.
+ *example_args (dataclasses or `argparse.ArgumentParser`): The arguments to the script. This function will only
+ try to extract the model and dataset name from those. Nothing else is tracked.
+ framework (`str`, *optional*, defaults to `"pytorch"`): The framework for the example.
+ """
+ if is_offline_mode():
+ return
+
+ data = {"example": example_name, "framework": framework}
+ for args in example_args:
+ args_as_dict = {k: v for k, v in args.__dict__.items() if not k.startswith("_") and v is not None}
+ if "model_name_or_path" in args_as_dict:
+ model_name = args_as_dict["model_name_or_path"]
+ # Filter out local paths
+ if not os.path.isdir(model_name):
+ data["model_name"] = args_as_dict["model_name_or_path"]
+ if "dataset_name" in args_as_dict:
+ data["dataset_name"] = args_as_dict["dataset_name"]
+ elif "task_name" in args_as_dict:
+ # Extract script name from the example_name
+ script_name = example_name.replace("tf_", "").replace("flax_", "").replace("run_", "")
+ script_name = script_name.replace("_no_trainer", "")
+ data["dataset_name"] = f"{script_name}-{args_as_dict['task_name']}"
+
+ # Send telemetry in the background
+ send_telemetry(
+ topic="examples", library_name="transformers", library_version=__version__, user_agent=http_user_agent(data)
+ )
+
+
+def convert_file_size_to_int(size: Union[int, str]):
+ """
+ Converts a size expressed as a string with digits an unit (like `"5MB"`) to an integer (in bytes).
+
+ Args:
+ size (`int` or `str`): The size to convert. Will be directly returned if an `int`.
+
+ Example:
+ ```py
+ >>> convert_file_size_to_int("1MiB")
+ 1048576
+ ```
+ """
+ if isinstance(size, int):
+ return size
+ if size.upper().endswith("GIB"):
+ return int(size[:-3]) * (2**30)
+ if size.upper().endswith("MIB"):
+ return int(size[:-3]) * (2**20)
+ if size.upper().endswith("KIB"):
+ return int(size[:-3]) * (2**10)
+ if size.upper().endswith("GB"):
+ int_size = int(size[:-2]) * (10**9)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("MB"):
+ int_size = int(size[:-2]) * (10**6)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("KB"):
+ int_size = int(size[:-2]) * (10**3)
+ return int_size // 8 if size.endswith("b") else int_size
+ raise ValueError("`size` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.")
+
+
+def get_checkpoint_shard_files(
+ pretrained_model_name_or_path,
+ index_filename,
+ cache_dir=None,
+ force_download=False,
+ proxies=None,
+ resume_download=None,
+ local_files_only=False,
+ token=None,
+ user_agent=None,
+ revision=None,
+ subfolder="",
+ _commit_hash=None,
+ **deprecated_kwargs,
+):
+ """
+ For a given model:
+
+ - download and cache all the shards of a sharded checkpoint if `pretrained_model_name_or_path` is a model ID on the
+ Hub
+ - returns the list of paths to all the shards, as well as some metadata.
+
+ For the description of each arg, see [`PreTrainedModel.from_pretrained`]. `index_filename` is the full path to the
+ index (downloaded and cached if `pretrained_model_name_or_path` is a model ID on the Hub).
+ """
+ import json
+
+ use_auth_token = deprecated_kwargs.pop("use_auth_token", None)
+ if use_auth_token is not None:
+ warnings.warn(
+ "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
+ FutureWarning,
+ )
+ if token is not None:
+ raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+ token = use_auth_token
+
+ if not os.path.isfile(index_filename):
+ raise ValueError(f"Can't find a checkpoint index ({index_filename}) in {pretrained_model_name_or_path}.")
+
+ with open(index_filename) as f:
+ index = json.loads(f.read())
+
+ shard_filenames = sorted(set(index["weight_map"].values()))
+ sharded_metadata = index["metadata"]
+ sharded_metadata["all_checkpoint_keys"] = list(index["weight_map"].keys())
+ sharded_metadata["weight_map"] = index["weight_map"].copy()
+
+ # First, let's deal with local folder.
+ if os.path.isdir(pretrained_model_name_or_path):
+ shard_filenames = [os.path.join(pretrained_model_name_or_path, subfolder, f) for f in shard_filenames]
+ return shard_filenames, sharded_metadata
+
+ # At this stage pretrained_model_name_or_path is a model identifier on the Hub. Try to get everything from cache,
+ # or download the files
+ cached_filenames = cached_files(
+ pretrained_model_name_or_path,
+ shard_filenames,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ proxies=proxies,
+ resume_download=resume_download,
+ local_files_only=local_files_only,
+ token=token,
+ user_agent=user_agent,
+ revision=revision,
+ subfolder=subfolder,
+ _commit_hash=_commit_hash,
+ )
+
+ return cached_filenames, sharded_metadata
+
+
+def create_and_tag_model_card(
+ repo_id: str,
+ tags: Optional[list[str]] = None,
+ token: Optional[str] = None,
+ ignore_metadata_errors: bool = False,
+):
+ """
+ Creates or loads an existing model card and tags it.
+
+ Args:
+ repo_id (`str`):
+ The repo_id where to look for the model card.
+ tags (`list[str]`, *optional*):
+ The list of tags to add in the model card
+ token (`str`, *optional*):
+ Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token.
+ ignore_metadata_errors (`bool`, *optional*, defaults to `False`):
+ If True, errors while parsing the metadata section will be ignored. Some information might be lost during
+ the process. Use it at your own risk.
+ """
+ try:
+ # Check if the model card is present on the remote repo
+ model_card = ModelCard.load(repo_id, token=token, ignore_metadata_errors=ignore_metadata_errors)
+ except EntryNotFoundError:
+ # Otherwise create a simple model card from template
+ model_description = "This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated."
+ card_data = ModelCardData(tags=[] if tags is None else tags, library_name="transformers")
+ model_card = ModelCard.from_template(card_data, model_description=model_description)
+
+ if tags is not None:
+ # Ensure model_card.data.tags is a list and not None
+ if model_card.data.tags is None:
+ model_card.data.tags = []
+ for model_tag in tags:
+ if model_tag not in model_card.data.tags:
+ model_card.data.tags.append(model_tag)
+
+ return model_card
+
+
+class PushInProgress:
+ """
+ Internal class to keep track of a push in progress (which might contain multiple `Future` jobs).
+ """
+
+ def __init__(self, jobs: Optional[futures.Future] = None) -> None:
+ self.jobs = [] if jobs is None else jobs
+
+ def is_done(self):
+ return all(job.done() for job in self.jobs)
+
+ def wait_until_done(self):
+ futures.wait(self.jobs)
+
+ def cancel(self) -> None:
+ self.jobs = [
+ job
+ for job in self.jobs
+ # Cancel the job if it wasn't started yet and remove cancelled/done jobs from the list
+ if not (job.cancel() or job.done())
+ ]
diff --git a/phivenv/Lib/site-packages/transformers/utils/import_utils.py b/phivenv/Lib/site-packages/transformers/utils/import_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..11c84566e3e8172f5390d37f42f0160dde2429f4
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/import_utils.py
@@ -0,0 +1,2883 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Import utilities: Utilities related to imports and our lazy inits.
+"""
+
+import importlib.machinery
+import importlib.metadata
+import importlib.util
+import json
+import operator
+import os
+import re
+import shutil
+import subprocess
+import sys
+import warnings
+from collections import OrderedDict
+from enum import Enum
+from functools import lru_cache
+from itertools import chain
+from types import ModuleType
+from typing import Any, Callable, Optional, Union
+
+from packaging import version
+
+from . import logging
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+# TODO: This doesn't work for all packages (`bs4`, `faiss`, etc.) Talk to Sylvain to see how to do with it better.
+def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[tuple[bool, str], bool]:
+ # Check if the package spec exists and grab its version to avoid importing a local directory
+ package_exists = importlib.util.find_spec(pkg_name) is not None
+ package_version = "N/A"
+ if package_exists:
+ try:
+ # TODO: Once python 3.9 support is dropped, `importlib.metadata.packages_distributions()`
+ # should be used here to map from package name to distribution names
+ # e.g. PIL -> Pillow, Pillow-SIMD; quark -> amd-quark; onnxruntime -> onnxruntime-gpu.
+ # `importlib.metadata.packages_distributions()` is not available in Python 3.9.
+
+ # Primary method to get the package version
+ package_version = importlib.metadata.version(pkg_name)
+ except importlib.metadata.PackageNotFoundError:
+ # Fallback method: Only for "torch" and versions containing "dev"
+ if pkg_name == "torch":
+ try:
+ package = importlib.import_module(pkg_name)
+ temp_version = getattr(package, "__version__", "N/A")
+ # Check if the version contains "dev"
+ if "dev" in temp_version:
+ package_version = temp_version
+ package_exists = True
+ else:
+ package_exists = False
+ except ImportError:
+ # If the package can't be imported, it's not available
+ package_exists = False
+ elif pkg_name == "quark":
+ # TODO: remove once `importlib.metadata.packages_distributions()` is supported.
+ try:
+ package_version = importlib.metadata.version("amd-quark")
+ except Exception:
+ package_exists = False
+ elif pkg_name == "triton":
+ try:
+ # import triton works for both linux and windows
+ package = importlib.import_module(pkg_name)
+ package_version = getattr(package, "__version__", "N/A")
+ except Exception:
+ try:
+ package_version = importlib.metadata.version("pytorch-triton") # pytorch-triton
+ except Exception:
+ package_exists = False
+ else:
+ # For packages other than "torch", don't attempt the fallback and set as not available
+ package_exists = False
+ logger.debug(f"Detected {pkg_name} version: {package_version}")
+ if return_version:
+ return package_exists, package_version
+ else:
+ return package_exists
+
+
+ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
+ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})
+
+USE_TF = os.environ.get("USE_TF", "AUTO").upper()
+USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
+USE_JAX = os.environ.get("USE_FLAX", "AUTO").upper()
+
+# Try to run a native pytorch job in an environment with TorchXLA installed by setting this value to 0.
+USE_TORCH_XLA = os.environ.get("USE_TORCH_XLA", "1").upper()
+
+FORCE_TF_AVAILABLE = os.environ.get("FORCE_TF_AVAILABLE", "AUTO").upper()
+
+# `transformers` requires `torch>=1.11` but this variable is exposed publicly, and we can't simply remove it.
+# This is the version of torch required to run torch.fx features and torch.onnx with dictionary inputs.
+TORCH_FX_REQUIRED_VERSION = version.parse("1.10")
+
+ACCELERATE_MIN_VERSION = "0.26.0"
+SCHEDULEFREE_MIN_VERSION = "1.2.6"
+FSDP_MIN_VERSION = "1.12.0"
+GGUF_MIN_VERSION = "0.10.0"
+XLA_FSDPV2_MIN_VERSION = "2.2.0"
+HQQ_MIN_VERSION = "0.2.1"
+VPTQ_MIN_VERSION = "0.0.4"
+TORCHAO_MIN_VERSION = "0.4.0"
+AUTOROUND_MIN_VERSION = "0.5.0"
+TRITON_MIN_VERSION = "1.0.0"
+
+_accelerate_available, _accelerate_version = _is_package_available("accelerate", return_version=True)
+_apex_available = _is_package_available("apex")
+_apollo_torch_available = _is_package_available("apollo_torch")
+_aqlm_available = _is_package_available("aqlm")
+_vptq_available, _vptq_version = _is_package_available("vptq", return_version=True)
+_av_available = importlib.util.find_spec("av") is not None
+_decord_available = importlib.util.find_spec("decord") is not None
+_torchcodec_available = importlib.util.find_spec("torchcodec") is not None
+_libcst_available = _is_package_available("libcst")
+_bitsandbytes_available = _is_package_available("bitsandbytes")
+_eetq_available = _is_package_available("eetq")
+_fbgemm_gpu_available = _is_package_available("fbgemm_gpu")
+_galore_torch_available = _is_package_available("galore_torch")
+_lomo_available = _is_package_available("lomo_optim")
+_grokadamw_available = _is_package_available("grokadamw")
+_schedulefree_available, _schedulefree_version = _is_package_available("schedulefree", return_version=True)
+_torch_optimi_available = importlib.util.find_spec("optimi") is not None
+# `importlib.metadata.version` doesn't work with `bs4` but `beautifulsoup4`. For `importlib.util.find_spec`, reversed.
+_bs4_available = importlib.util.find_spec("bs4") is not None
+_coloredlogs_available = _is_package_available("coloredlogs")
+# `importlib.metadata.util` doesn't work with `opencv-python-headless`.
+_cv2_available = importlib.util.find_spec("cv2") is not None
+_yt_dlp_available = importlib.util.find_spec("yt_dlp") is not None
+_datasets_available = _is_package_available("datasets")
+_detectron2_available = _is_package_available("detectron2")
+# We need to check `faiss`, `faiss-cpu` and `faiss-gpu`.
+_faiss_available = importlib.util.find_spec("faiss") is not None
+try:
+ _faiss_version = importlib.metadata.version("faiss")
+ logger.debug(f"Successfully imported faiss version {_faiss_version}")
+except importlib.metadata.PackageNotFoundError:
+ try:
+ _faiss_version = importlib.metadata.version("faiss-cpu")
+ logger.debug(f"Successfully imported faiss version {_faiss_version}")
+ except importlib.metadata.PackageNotFoundError:
+ try:
+ _faiss_version = importlib.metadata.version("faiss-gpu")
+ logger.debug(f"Successfully imported faiss version {_faiss_version}")
+ except importlib.metadata.PackageNotFoundError:
+ _faiss_available = False
+_ftfy_available = _is_package_available("ftfy")
+_g2p_en_available = _is_package_available("g2p_en")
+_hadamard_available = _is_package_available("fast_hadamard_transform")
+_ipex_available, _ipex_version = _is_package_available("intel_extension_for_pytorch", return_version=True)
+_jieba_available = _is_package_available("jieba")
+_jinja_available = _is_package_available("jinja2")
+_kenlm_available = _is_package_available("kenlm")
+_keras_nlp_available = _is_package_available("keras_nlp")
+_levenshtein_available = _is_package_available("Levenshtein")
+_librosa_available = _is_package_available("librosa")
+_natten_available = _is_package_available("natten")
+_nltk_available = _is_package_available("nltk")
+_onnx_available = _is_package_available("onnx")
+_openai_available = _is_package_available("openai")
+_optimum_available = _is_package_available("optimum")
+_auto_gptq_available = _is_package_available("auto_gptq")
+_gptqmodel_available = _is_package_available("gptqmodel")
+_auto_round_available, _auto_round_version = _is_package_available("auto_round", return_version=True)
+# `importlib.metadata.version` doesn't work with `awq`
+_auto_awq_available = importlib.util.find_spec("awq") is not None
+_quark_available = _is_package_available("quark")
+_fp_quant_available, _fp_quant_version = _is_package_available("fp_quant", return_version=True)
+_qutlass_available = _is_package_available("qutlass")
+_is_optimum_quanto_available = False
+try:
+ importlib.metadata.version("optimum_quanto")
+ _is_optimum_quanto_available = True
+except importlib.metadata.PackageNotFoundError:
+ _is_optimum_quanto_available = False
+# For compressed_tensors, only check spec to allow compressed_tensors-nightly package
+_compressed_tensors_available = importlib.util.find_spec("compressed_tensors") is not None
+_pandas_available = _is_package_available("pandas")
+_peft_available = _is_package_available("peft")
+_phonemizer_available = _is_package_available("phonemizer")
+_uroman_available = _is_package_available("uroman")
+_psutil_available = _is_package_available("psutil")
+_py3nvml_available = _is_package_available("py3nvml")
+_pyctcdecode_available = _is_package_available("pyctcdecode")
+_pygments_available = _is_package_available("pygments")
+_pytesseract_available = _is_package_available("pytesseract")
+_pytest_available = _is_package_available("pytest")
+_pytorch_quantization_available = _is_package_available("pytorch_quantization")
+_rjieba_available = _is_package_available("rjieba")
+_sacremoses_available = _is_package_available("sacremoses")
+_safetensors_available = _is_package_available("safetensors")
+_scipy_available = _is_package_available("scipy")
+_sentencepiece_available = _is_package_available("sentencepiece")
+_is_seqio_available = _is_package_available("seqio")
+_is_gguf_available, _gguf_version = _is_package_available("gguf", return_version=True)
+_sklearn_available = importlib.util.find_spec("sklearn") is not None
+if _sklearn_available:
+ try:
+ importlib.metadata.version("scikit-learn")
+ except importlib.metadata.PackageNotFoundError:
+ _sklearn_available = False
+_smdistributed_available = importlib.util.find_spec("smdistributed") is not None
+_soundfile_available = _is_package_available("soundfile")
+_spacy_available = _is_package_available("spacy")
+_sudachipy_available, _sudachipy_version = _is_package_available("sudachipy", return_version=True)
+_tensorflow_probability_available = _is_package_available("tensorflow_probability")
+_tensorflow_text_available = _is_package_available("tensorflow_text")
+_tf2onnx_available = _is_package_available("tf2onnx")
+_timm_available = _is_package_available("timm")
+_tokenizers_available = _is_package_available("tokenizers")
+_torchaudio_available = _is_package_available("torchaudio")
+_torchao_available, _torchao_version = _is_package_available("torchao", return_version=True)
+_torchdistx_available = _is_package_available("torchdistx")
+_torchvision_available, _torchvision_version = _is_package_available("torchvision", return_version=True)
+_mlx_available = _is_package_available("mlx")
+_num2words_available = _is_package_available("num2words")
+_hqq_available, _hqq_version = _is_package_available("hqq", return_version=True)
+_tiktoken_available = _is_package_available("tiktoken")
+_blobfile_available = _is_package_available("blobfile")
+_liger_kernel_available = _is_package_available("liger_kernel")
+_spqr_available = _is_package_available("spqr_quant")
+_rich_available = _is_package_available("rich")
+_kernels_available = _is_package_available("kernels")
+_matplotlib_available = _is_package_available("matplotlib")
+_mistral_common_available = _is_package_available("mistral_common")
+_triton_available, _triton_version = _is_package_available("triton", return_version=True)
+
+_torch_version = "N/A"
+_torch_available = False
+if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
+ _torch_available, _torch_version = _is_package_available("torch", return_version=True)
+ if _torch_available:
+ _torch_available = version.parse(_torch_version) >= version.parse("2.1.0")
+ if not _torch_available:
+ logger.warning(f"Disabling PyTorch because PyTorch >= 2.1 is required but found {_torch_version}")
+else:
+ logger.info("Disabling PyTorch because USE_TF is set")
+ _torch_available = False
+
+
+_tf_version = "N/A"
+_tf_available = False
+if FORCE_TF_AVAILABLE in ENV_VARS_TRUE_VALUES:
+ _tf_available = True
+else:
+ if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
+ # Note: _is_package_available("tensorflow") fails for tensorflow-cpu. Please test any changes to the line below
+ # with tensorflow-cpu to make sure it still works!
+ _tf_available = importlib.util.find_spec("tensorflow") is not None
+ if _tf_available:
+ candidates = (
+ "tensorflow",
+ "tensorflow-cpu",
+ "tensorflow-gpu",
+ "tf-nightly",
+ "tf-nightly-cpu",
+ "tf-nightly-gpu",
+ "tf-nightly-rocm",
+ "intel-tensorflow",
+ "intel-tensorflow-avx512",
+ "tensorflow-rocm",
+ "tensorflow-macos",
+ "tensorflow-aarch64",
+ )
+ _tf_version = None
+ # For the metadata, we have to look for both tensorflow and tensorflow-cpu
+ for pkg in candidates:
+ try:
+ _tf_version = importlib.metadata.version(pkg)
+ break
+ except importlib.metadata.PackageNotFoundError:
+ pass
+ _tf_available = _tf_version is not None
+ if _tf_available:
+ if version.parse(_tf_version) < version.parse("2"):
+ logger.info(
+ f"TensorFlow found but with version {_tf_version}. Transformers requires version 2 minimum."
+ )
+ _tf_available = False
+ else:
+ logger.info("Disabling Tensorflow because USE_TORCH is set")
+
+
+_essentia_available = importlib.util.find_spec("essentia") is not None
+try:
+ _essentia_version = importlib.metadata.version("essentia")
+ logger.debug(f"Successfully imported essentia version {_essentia_version}")
+except importlib.metadata.PackageNotFoundError:
+ _essentia_version = False
+
+
+_pydantic_available = importlib.util.find_spec("pydantic") is not None
+try:
+ _pydantic_version = importlib.metadata.version("pydantic")
+ logger.debug(f"Successfully imported pydantic version {_pydantic_version}")
+except importlib.metadata.PackageNotFoundError:
+ _pydantic_available = False
+
+
+_fastapi_available = importlib.util.find_spec("fastapi") is not None
+try:
+ _fastapi_version = importlib.metadata.version("fastapi")
+ logger.debug(f"Successfully imported pydantic version {_fastapi_version}")
+except importlib.metadata.PackageNotFoundError:
+ _fastapi_available = False
+
+
+_uvicorn_available = importlib.util.find_spec("uvicorn") is not None
+try:
+ _uvicorn_version = importlib.metadata.version("uvicorn")
+ logger.debug(f"Successfully imported pydantic version {_uvicorn_version}")
+except importlib.metadata.PackageNotFoundError:
+ _uvicorn_available = False
+
+
+_pretty_midi_available = importlib.util.find_spec("pretty_midi") is not None
+try:
+ _pretty_midi_version = importlib.metadata.version("pretty_midi")
+ logger.debug(f"Successfully imported pretty_midi version {_pretty_midi_version}")
+except importlib.metadata.PackageNotFoundError:
+ _pretty_midi_available = False
+
+
+ccl_version = "N/A"
+_is_ccl_available = (
+ importlib.util.find_spec("torch_ccl") is not None
+ or importlib.util.find_spec("oneccl_bindings_for_pytorch") is not None
+)
+try:
+ ccl_version = importlib.metadata.version("oneccl_bind_pt")
+ logger.debug(f"Detected oneccl_bind_pt version {ccl_version}")
+except importlib.metadata.PackageNotFoundError:
+ _is_ccl_available = False
+
+
+_flax_available = False
+if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
+ _flax_available, _flax_version = _is_package_available("flax", return_version=True)
+ if _flax_available:
+ _jax_available, _jax_version = _is_package_available("jax", return_version=True)
+ if _jax_available:
+ logger.info(f"JAX version {_jax_version}, Flax version {_flax_version} available.")
+ else:
+ _flax_available = _jax_available = False
+ _jax_version = _flax_version = "N/A"
+
+
+_torch_xla_available = False
+if USE_TORCH_XLA in ENV_VARS_TRUE_VALUES:
+ _torch_xla_available, _torch_xla_version = _is_package_available("torch_xla", return_version=True)
+ if _torch_xla_available:
+ logger.info(f"Torch XLA version {_torch_xla_version} available.")
+
+
+def is_kenlm_available() -> Union[tuple[bool, str], bool]:
+ return _kenlm_available
+
+
+def is_kernels_available() -> Union[tuple[bool, str], bool]:
+ return _kernels_available
+
+
+def is_cv2_available() -> Union[tuple[bool, str], bool]:
+ return _cv2_available
+
+
+def is_yt_dlp_available() -> Union[tuple[bool, str], bool]:
+ return _yt_dlp_available
+
+
+def is_torch_available() -> Union[tuple[bool, str], bool]:
+ return _torch_available
+
+
+def is_libcst_available() -> Union[tuple[bool, str], bool]:
+ return _libcst_available
+
+
+def is_accelerate_available(min_version: str = ACCELERATE_MIN_VERSION) -> bool:
+ return _accelerate_available and version.parse(_accelerate_version) >= version.parse(min_version)
+
+
+def is_torch_accelerator_available() -> bool:
+ if is_torch_available():
+ import torch
+
+ return hasattr(torch, "accelerator")
+
+ return False
+
+
+def is_torch_deterministic() -> bool:
+ """
+ Check whether pytorch uses deterministic algorithms by looking if torch.set_deterministic_debug_mode() is set to 1 or 2"
+ """
+ if is_torch_available():
+ import torch
+
+ if torch.get_deterministic_debug_mode() == 0:
+ return False
+ else:
+ return True
+
+ return False
+
+
+def is_triton_available(min_version: str = TRITON_MIN_VERSION) -> bool:
+ return _triton_available and version.parse(_triton_version) >= version.parse(min_version)
+
+
+def is_hadamard_available() -> Union[tuple[bool, str], bool]:
+ return _hadamard_available
+
+
+def is_hqq_available(min_version: str = HQQ_MIN_VERSION) -> bool:
+ return _hqq_available and version.parse(_hqq_version) >= version.parse(min_version)
+
+
+def is_pygments_available() -> Union[tuple[bool, str], bool]:
+ return _pygments_available
+
+
+def get_torch_version() -> str:
+ return _torch_version
+
+
+def get_torch_major_and_minor_version() -> str:
+ if _torch_version == "N/A":
+ return "N/A"
+ parsed_version = version.parse(_torch_version)
+ return str(parsed_version.major) + "." + str(parsed_version.minor)
+
+
+def is_torch_sdpa_available():
+ # Mostly retained for backward compatibility in remote code, since sdpa works correctly on all torch versions >= 2.2
+ if not is_torch_available() or _torch_version == "N/A":
+ return False
+ return True
+
+
+def is_torch_flex_attn_available() -> bool:
+ if not is_torch_available() or _torch_version == "N/A":
+ return False
+
+ # TODO check if some bugs cause push backs on the exact version
+ # NOTE: We require torch>=2.5.0 as it is the first release
+ return version.parse(_torch_version) >= version.parse("2.5.0")
+
+
+def is_torchvision_available() -> bool:
+ return _torchvision_available
+
+
+def is_torchvision_v2_available() -> bool:
+ if not is_torchvision_available():
+ return False
+
+ # NOTE: We require torchvision>=0.15 as v2 transforms are available from this version: https://pytorch.org/vision/stable/transforms.html#v1-or-v2-which-one-should-i-use
+ return version.parse(_torchvision_version) >= version.parse("0.15")
+
+
+def is_galore_torch_available() -> Union[tuple[bool, str], bool]:
+ return _galore_torch_available
+
+
+def is_apollo_torch_available() -> Union[tuple[bool, str], bool]:
+ return _apollo_torch_available
+
+
+def is_torch_optimi_available() -> Union[tuple[bool, str], bool]:
+ return _torch_optimi_available
+
+
+def is_lomo_available() -> Union[tuple[bool, str], bool]:
+ return _lomo_available
+
+
+def is_grokadamw_available() -> Union[tuple[bool, str], bool]:
+ return _grokadamw_available
+
+
+def is_schedulefree_available(min_version: str = SCHEDULEFREE_MIN_VERSION) -> bool:
+ return _schedulefree_available and version.parse(_schedulefree_version) >= version.parse(min_version)
+
+
+def is_pyctcdecode_available() -> Union[tuple[bool, str], bool]:
+ return _pyctcdecode_available
+
+
+def is_librosa_available() -> Union[tuple[bool, str], bool]:
+ return _librosa_available
+
+
+def is_essentia_available() -> Union[tuple[bool, str], bool]:
+ return _essentia_available
+
+
+def is_pydantic_available() -> Union[tuple[bool, str], bool]:
+ return _pydantic_available
+
+
+def is_fastapi_available() -> Union[tuple[bool, str], bool]:
+ return _fastapi_available
+
+
+def is_uvicorn_available() -> Union[tuple[bool, str], bool]:
+ return _uvicorn_available
+
+
+def is_openai_available() -> Union[tuple[bool, str], bool]:
+ return _openai_available
+
+
+def is_pretty_midi_available() -> Union[tuple[bool, str], bool]:
+ return _pretty_midi_available
+
+
+def is_torch_cuda_available() -> bool:
+ if is_torch_available():
+ import torch
+
+ return torch.cuda.is_available()
+ else:
+ return False
+
+
+def is_cuda_platform() -> bool:
+ if is_torch_available():
+ import torch
+
+ return torch.version.cuda is not None
+ else:
+ return False
+
+
+def is_rocm_platform() -> bool:
+ if is_torch_available():
+ import torch
+
+ return torch.version.hip is not None
+ else:
+ return False
+
+
+def is_mamba_ssm_available() -> Union[tuple[bool, str], bool]:
+ if is_torch_available():
+ import torch
+
+ if not torch.cuda.is_available():
+ return False
+ else:
+ return _is_package_available("mamba_ssm")
+ return False
+
+
+def is_mamba_2_ssm_available() -> bool:
+ if is_torch_available():
+ import torch
+
+ if not torch.cuda.is_available():
+ return False
+ else:
+ if _is_package_available("mamba_ssm"):
+ import mamba_ssm
+
+ if version.parse(mamba_ssm.__version__) >= version.parse("2.0.4"):
+ return True
+ return False
+
+
+def is_causal_conv1d_available() -> Union[tuple[bool, str], bool]:
+ if is_torch_available():
+ import torch
+
+ if not torch.cuda.is_available():
+ return False
+ return _is_package_available("causal_conv1d")
+ return False
+
+
+def is_xlstm_available() -> Union[tuple[bool, str], bool]:
+ if is_torch_available():
+ return _is_package_available("xlstm")
+ return False
+
+
+def is_mambapy_available() -> Union[tuple[bool, str], bool]:
+ if is_torch_available():
+ return _is_package_available("mambapy")
+ return False
+
+
+def is_torch_mps_available(min_version: Optional[str] = None) -> bool:
+ if is_torch_available():
+ import torch
+
+ if hasattr(torch.backends, "mps"):
+ backend_available = torch.backends.mps.is_available() and torch.backends.mps.is_built()
+ if min_version is not None:
+ flag = version.parse(_torch_version) >= version.parse(min_version)
+ backend_available = backend_available and flag
+ return backend_available
+ return False
+
+
+def is_torch_bf16_gpu_available() -> bool:
+ if not is_torch_available():
+ return False
+
+ import torch
+
+ if torch.cuda.is_available():
+ return torch.cuda.is_bf16_supported()
+ if is_torch_xpu_available():
+ return torch.xpu.is_bf16_supported()
+ if is_torch_hpu_available():
+ return True
+ if is_torch_npu_available():
+ return torch.npu.is_bf16_supported()
+ if is_torch_mps_available():
+ # Note: Emulated in software by Metal using fp32 for hardware without native support (like M1/M2)
+ return torch.backends.mps.is_macos_or_newer(14, 0)
+ return False
+
+
+def is_torch_bf16_cpu_available() -> Union[tuple[bool, str], bool]:
+ return is_torch_available()
+
+
+def is_torch_bf16_available() -> bool:
+ # the original bf16 check was for gpu only, but later a cpu/bf16 combo has emerged so this util
+ # has become ambiguous and therefore deprecated
+ warnings.warn(
+ "The util is_torch_bf16_available is deprecated, please use is_torch_bf16_gpu_available "
+ "or is_torch_bf16_cpu_available instead according to whether it's used with cpu or gpu",
+ FutureWarning,
+ )
+ return is_torch_bf16_gpu_available()
+
+
+@lru_cache
+def is_torch_fp16_available_on_device(device: str) -> bool:
+ if not is_torch_available():
+ return False
+
+ if is_torch_hpu_available():
+ if is_habana_gaudi1():
+ return False
+ else:
+ return True
+
+ import torch
+
+ try:
+ x = torch.zeros(2, 2, dtype=torch.float16, device=device)
+ _ = x @ x
+
+ # At this moment, let's be strict of the check: check if `LayerNorm` is also supported on device, because many
+ # models use this layer.
+ batch, sentence_length, embedding_dim = 3, 4, 5
+ embedding = torch.randn(batch, sentence_length, embedding_dim, dtype=torch.float16, device=device)
+ layer_norm = torch.nn.LayerNorm(embedding_dim, dtype=torch.float16, device=device)
+ _ = layer_norm(embedding)
+
+ except: # noqa: E722
+ # TODO: more precise exception matching, if possible.
+ # most backends should return `RuntimeError` however this is not guaranteed.
+ return False
+
+ return True
+
+
+@lru_cache
+def is_torch_bf16_available_on_device(device: str) -> bool:
+ if not is_torch_available():
+ return False
+
+ import torch
+
+ if device == "cuda":
+ return is_torch_bf16_gpu_available()
+
+ if device == "hpu":
+ return True
+
+ try:
+ x = torch.zeros(2, 2, dtype=torch.bfloat16, device=device)
+ _ = x @ x
+ except: # noqa: E722
+ # TODO: more precise exception matching, if possible.
+ # most backends should return `RuntimeError` however this is not guaranteed.
+ return False
+
+ return True
+
+
+def is_torch_tf32_available() -> bool:
+ if not is_torch_available():
+ return False
+
+ import torch
+
+ if not torch.cuda.is_available() or torch.version.cuda is None:
+ return False
+ if torch.cuda.get_device_properties(torch.cuda.current_device()).major < 8:
+ return False
+ return True
+
+
+def is_torch_fx_available() -> Union[tuple[bool, str], bool]:
+ return is_torch_available()
+
+
+def is_peft_available() -> Union[tuple[bool, str], bool]:
+ return _peft_available
+
+
+def is_bs4_available() -> Union[tuple[bool, str], bool]:
+ return _bs4_available
+
+
+def is_tf_available() -> bool:
+ return _tf_available
+
+
+def is_coloredlogs_available() -> Union[tuple[bool, str], bool]:
+ return _coloredlogs_available
+
+
+def is_tf2onnx_available() -> Union[tuple[bool, str], bool]:
+ return _tf2onnx_available
+
+
+def is_onnx_available() -> Union[tuple[bool, str], bool]:
+ return _onnx_available
+
+
+def is_flax_available() -> bool:
+ return _flax_available
+
+
+def is_flute_available() -> bool:
+ try:
+ return importlib.util.find_spec("flute") is not None and importlib.metadata.version("flute-kernel") >= "0.4.1"
+ except importlib.metadata.PackageNotFoundError:
+ return False
+
+
+def is_ftfy_available() -> Union[tuple[bool, str], bool]:
+ return _ftfy_available
+
+
+def is_g2p_en_available() -> Union[tuple[bool, str], bool]:
+ return _g2p_en_available
+
+
+@lru_cache
+def is_torch_xla_available(check_is_tpu=False, check_is_gpu=False) -> bool:
+ """
+ Check if `torch_xla` is available. To train a native pytorch job in an environment with torch xla installed, set
+ the USE_TORCH_XLA to false.
+ """
+ assert not (check_is_tpu and check_is_gpu), "The check_is_tpu and check_is_gpu cannot both be true."
+
+ if not _torch_xla_available:
+ return False
+
+ import torch_xla
+
+ if check_is_gpu:
+ return torch_xla.runtime.device_type() in ["GPU", "CUDA"]
+ elif check_is_tpu:
+ return torch_xla.runtime.device_type() == "TPU"
+
+ return True
+
+
+@lru_cache
+def is_torch_neuroncore_available(check_device=True) -> bool:
+ if importlib.util.find_spec("torch_neuronx") is not None:
+ return is_torch_xla_available()
+ return False
+
+
+@lru_cache
+def is_torch_npu_available(check_device=False) -> bool:
+ "Checks if `torch_npu` is installed and potentially if a NPU is in the environment"
+ if not _torch_available or importlib.util.find_spec("torch_npu") is None:
+ return False
+
+ import torch
+ import torch_npu # noqa: F401
+
+ if check_device:
+ try:
+ # Will raise a RuntimeError if no NPU is found
+ _ = torch.npu.device_count()
+ return torch.npu.is_available()
+ except RuntimeError:
+ return False
+ return hasattr(torch, "npu") and torch.npu.is_available()
+
+
+@lru_cache
+def is_torch_mlu_available(check_device=False) -> bool:
+ """
+ Checks if `mlu` is available via an `cndev-based` check which won't trigger the drivers and leave mlu
+ uninitialized.
+ """
+ if not _torch_available or importlib.util.find_spec("torch_mlu") is None:
+ return False
+
+ import torch
+ import torch_mlu # noqa: F401
+
+ pytorch_cndev_based_mlu_check_previous_value = os.environ.get("PYTORCH_CNDEV_BASED_MLU_CHECK")
+ try:
+ os.environ["PYTORCH_CNDEV_BASED_MLU_CHECK"] = str(1)
+ available = torch.mlu.is_available()
+ finally:
+ if pytorch_cndev_based_mlu_check_previous_value:
+ os.environ["PYTORCH_CNDEV_BASED_MLU_CHECK"] = pytorch_cndev_based_mlu_check_previous_value
+ else:
+ os.environ.pop("PYTORCH_CNDEV_BASED_MLU_CHECK", None)
+
+ return available
+
+
+@lru_cache
+def is_torch_musa_available(check_device=False) -> bool:
+ "Checks if `torch_musa` is installed and potentially if a MUSA is in the environment"
+ if not _torch_available or importlib.util.find_spec("torch_musa") is None:
+ return False
+
+ import torch
+ import torch_musa # noqa: F401
+
+ torch_musa_min_version = "0.33.0"
+ if _accelerate_available and version.parse(_accelerate_version) < version.parse(torch_musa_min_version):
+ return False
+
+ if check_device:
+ try:
+ # Will raise a RuntimeError if no MUSA is found
+ _ = torch.musa.device_count()
+ return torch.musa.is_available()
+ except RuntimeError:
+ return False
+ return hasattr(torch, "musa") and torch.musa.is_available()
+
+
+@lru_cache
+def is_torch_hpu_available() -> bool:
+ "Checks if `torch.hpu` is available and potentially if a HPU is in the environment"
+ if (
+ not _torch_available
+ or importlib.util.find_spec("habana_frameworks") is None
+ or importlib.util.find_spec("habana_frameworks.torch") is None
+ ):
+ return False
+
+ torch_hpu_min_accelerate_version = "1.5.0"
+ if _accelerate_available and version.parse(_accelerate_version) < version.parse(torch_hpu_min_accelerate_version):
+ return False
+
+ import torch
+
+ if os.environ.get("PT_HPU_LAZY_MODE", "1") == "1":
+ # import habana_frameworks.torch in case of lazy mode to patch torch with torch.hpu
+ import habana_frameworks.torch # noqa: F401
+
+ if not hasattr(torch, "hpu") or not torch.hpu.is_available():
+ return False
+
+ # We patch torch.gather for int64 tensors to avoid a bug on Gaudi
+ # Graph compile failed with synStatus 26 [Generic failure]
+ # This can be removed once bug is fixed but for now we need it.
+ original_gather = torch.gather
+
+ def patched_gather(input: torch.Tensor, dim: int, index: torch.LongTensor) -> torch.Tensor:
+ if input.dtype == torch.int64 and input.device.type == "hpu":
+ return original_gather(input.to(torch.int32), dim, index).to(torch.int64)
+ else:
+ return original_gather(input, dim, index)
+
+ torch.gather = patched_gather
+ torch.Tensor.gather = patched_gather
+
+ original_take_along_dim = torch.take_along_dim
+
+ def patched_take_along_dim(
+ input: torch.Tensor, indices: torch.LongTensor, dim: Optional[int] = None
+ ) -> torch.Tensor:
+ if input.dtype == torch.int64 and input.device.type == "hpu":
+ return original_take_along_dim(input.to(torch.int32), indices, dim).to(torch.int64)
+ else:
+ return original_take_along_dim(input, indices, dim)
+
+ torch.take_along_dim = patched_take_along_dim
+
+ original_cholesky = torch.linalg.cholesky
+
+ def safe_cholesky(A, *args, **kwargs):
+ output = original_cholesky(A, *args, **kwargs)
+
+ if torch.isnan(output).any():
+ jitter_value = 1e-9
+ diag_jitter = torch.eye(A.size(-1), dtype=A.dtype, device=A.device) * jitter_value
+ output = original_cholesky(A + diag_jitter, *args, **kwargs)
+
+ return output
+
+ torch.linalg.cholesky = safe_cholesky
+
+ original_scatter = torch.scatter
+
+ def patched_scatter(
+ input: torch.Tensor, dim: int, index: torch.Tensor, src: torch.Tensor, *args, **kwargs
+ ) -> torch.Tensor:
+ if input.device.type == "hpu" and input is src:
+ return original_scatter(input, dim, index, src.clone(), *args, **kwargs)
+ else:
+ return original_scatter(input, dim, index, src, *args, **kwargs)
+
+ torch.scatter = patched_scatter
+ torch.Tensor.scatter = patched_scatter
+
+ # IlyasMoutawwakil: we patch torch.compile to use the HPU backend by default
+ # https://github.com/huggingface/transformers/pull/38790#discussion_r2157043944
+ # This is necessary for cases where torch.compile is used as a decorator (defaulting to inductor)
+ # https://github.com/huggingface/transformers/blob/af6120b3eb2470b994c21421bb6eaa76576128b0/src/transformers/models/modernbert/modeling_modernbert.py#L204
+ original_compile = torch.compile
+
+ def hpu_backend_compile(*args, **kwargs):
+ if kwargs.get("backend") not in ["hpu_backend", "eager"]:
+ logger.warning(
+ f"Calling torch.compile with backend={kwargs.get('backend')} on a Gaudi device is not supported. "
+ "We will override the backend with 'hpu_backend' to avoid errors."
+ )
+ kwargs["backend"] = "hpu_backend"
+
+ return original_compile(*args, **kwargs)
+
+ torch.compile = hpu_backend_compile
+
+ return True
+
+
+@lru_cache
+def is_habana_gaudi1() -> bool:
+ if not is_torch_hpu_available():
+ return False
+
+ import habana_frameworks.torch.utils.experimental as htexp # noqa: F401
+
+ # Check if the device is Gaudi1 (vs Gaudi2, Gaudi3)
+ return htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi
+
+
+def is_torchdynamo_available() -> Union[tuple[bool, str], bool]:
+ return is_torch_available()
+
+
+def is_torch_compile_available() -> Union[tuple[bool, str], bool]:
+ return is_torch_available()
+
+
+def is_torchdynamo_compiling() -> Union[tuple[bool, str], bool]:
+ if not is_torch_available():
+ return False
+
+ # Importing torch._dynamo causes issues with PyTorch profiler (https://github.com/pytorch/pytorch/issues/130622)
+ # hence rather relying on `torch.compiler.is_compiling()` when possible (torch>=2.3)
+ try:
+ import torch
+
+ return torch.compiler.is_compiling()
+ except Exception:
+ try:
+ import torch._dynamo as dynamo # noqa: F401
+
+ return dynamo.is_compiling()
+ except Exception:
+ return False
+
+
+def is_torchdynamo_exporting() -> bool:
+ if not is_torch_available():
+ return False
+
+ try:
+ import torch
+
+ return torch.compiler.is_exporting()
+ except Exception:
+ try:
+ import torch._dynamo as dynamo # noqa: F401
+
+ return dynamo.is_exporting()
+ except Exception:
+ return False
+
+
+def is_torch_tensorrt_fx_available() -> bool:
+ if importlib.util.find_spec("torch_tensorrt") is None:
+ return False
+ return importlib.util.find_spec("torch_tensorrt.fx") is not None
+
+
+def is_datasets_available() -> Union[tuple[bool, str], bool]:
+ return _datasets_available
+
+
+def is_detectron2_available() -> Union[tuple[bool, str], bool]:
+ return _detectron2_available
+
+
+def is_rjieba_available() -> Union[tuple[bool, str], bool]:
+ return _rjieba_available
+
+
+def is_psutil_available() -> Union[tuple[bool, str], bool]:
+ return _psutil_available
+
+
+def is_py3nvml_available() -> Union[tuple[bool, str], bool]:
+ return _py3nvml_available
+
+
+def is_sacremoses_available() -> Union[tuple[bool, str], bool]:
+ return _sacremoses_available
+
+
+def is_apex_available() -> Union[tuple[bool, str], bool]:
+ return _apex_available
+
+
+def is_aqlm_available() -> Union[tuple[bool, str], bool]:
+ return _aqlm_available
+
+
+def is_vptq_available(min_version: str = VPTQ_MIN_VERSION) -> bool:
+ return _vptq_available and version.parse(_vptq_version) >= version.parse(min_version)
+
+
+def is_av_available() -> bool:
+ return _av_available
+
+
+def is_decord_available() -> bool:
+ return _decord_available
+
+
+def is_torchcodec_available() -> bool:
+ return _torchcodec_available
+
+
+def is_ninja_available() -> bool:
+ r"""
+ Code comes from *torch.utils.cpp_extension.is_ninja_available()*. Returns `True` if the
+ [ninja](https://ninja-build.org/) build system is available on the system, `False` otherwise.
+ """
+ try:
+ subprocess.check_output(["ninja", "--version"])
+ except Exception:
+ return False
+ else:
+ return True
+
+
+def is_ipex_available(min_version: str = "") -> bool:
+ def get_major_and_minor_from_version(full_version):
+ return str(version.parse(full_version).major) + "." + str(version.parse(full_version).minor)
+
+ if not is_torch_available() or not _ipex_available:
+ return False
+
+ torch_major_and_minor = get_major_and_minor_from_version(_torch_version)
+ ipex_major_and_minor = get_major_and_minor_from_version(_ipex_version)
+ if torch_major_and_minor != ipex_major_and_minor:
+ logger.warning(
+ f"Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"
+ f" but PyTorch {_torch_version} is found. Please switch to the matching version and run again."
+ )
+ return False
+ if min_version:
+ return version.parse(_ipex_version) >= version.parse(min_version)
+ return True
+
+
+@lru_cache
+def is_torch_xpu_available(check_device: bool = False) -> bool:
+ """
+ Checks if XPU acceleration is available either via native PyTorch (>=2.6),
+ `intel_extension_for_pytorch` or via stock PyTorch (>=2.4) and potentially
+ if a XPU is in the environment.
+ """
+ if not is_torch_available():
+ return False
+
+ torch_version = version.parse(_torch_version)
+ if torch_version.major == 2 and torch_version.minor < 6:
+ if is_ipex_available():
+ import intel_extension_for_pytorch # noqa: F401
+ elif torch_version.major == 2 and torch_version.minor < 4:
+ return False
+
+ import torch
+
+ if check_device:
+ try:
+ # Will raise a RuntimeError if no XPU is found
+ _ = torch.xpu.device_count()
+ return torch.xpu.is_available()
+ except RuntimeError:
+ return False
+ return hasattr(torch, "xpu") and torch.xpu.is_available()
+
+
+@lru_cache
+def is_bitsandbytes_available(check_library_only: bool = False) -> bool:
+ if not _bitsandbytes_available:
+ return False
+
+ if check_library_only:
+ return True
+
+ if not is_torch_available():
+ return False
+
+ import torch
+
+ # `bitsandbytes` versions older than 0.43.1 eagerly require CUDA at import time,
+ # so those versions of the library are practically only available when CUDA is too.
+ if version.parse(importlib.metadata.version("bitsandbytes")) < version.parse("0.43.1"):
+ return torch.cuda.is_available()
+
+ # Newer versions of `bitsandbytes` can be imported on systems without CUDA.
+ return True
+
+
+def is_bitsandbytes_multi_backend_available() -> bool:
+ if not is_bitsandbytes_available():
+ return False
+
+ import bitsandbytes as bnb
+
+ return "multi_backend" in getattr(bnb, "features", set())
+
+
+def is_flash_attn_2_available() -> bool:
+ if not is_torch_available():
+ return False
+
+ if not _is_package_available("flash_attn"):
+ return False
+
+ # Let's add an extra check to see if cuda is available
+ import torch
+
+ if not (torch.cuda.is_available() or is_torch_mlu_available()):
+ return False
+
+ if torch.version.cuda:
+ return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.1.0")
+ elif torch.version.hip:
+ # TODO: Bump the requirement to 2.1.0 once released in https://github.com/ROCmSoftwarePlatform/flash-attention
+ return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.0.4")
+ elif is_torch_mlu_available():
+ return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.3.3")
+ else:
+ return False
+
+
+@lru_cache
+def is_flash_attn_3_available() -> bool:
+ if not is_torch_available():
+ return False
+
+ if not _is_package_available("flash_attn_3"):
+ return False
+
+ import torch
+
+ if not torch.cuda.is_available():
+ return False
+
+ # TODO: Check for a minimum version when FA3 is stable
+ # return version.parse(importlib.metadata.version("flash_attn_3")) >= version.parse("3.0.0")
+
+ return True
+
+
+@lru_cache
+def is_flash_attn_greater_or_equal_2_10() -> bool:
+ if not _is_package_available("flash_attn"):
+ return False
+
+ return version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.1.0")
+
+
+@lru_cache
+def is_flash_attn_greater_or_equal(library_version: str) -> bool:
+ if not _is_package_available("flash_attn"):
+ return False
+
+ return version.parse(importlib.metadata.version("flash_attn")) >= version.parse(library_version)
+
+
+@lru_cache
+def is_torch_greater_or_equal(library_version: str, accept_dev: bool = False) -> bool:
+ """
+ Accepts a library version and returns True if the current version of the library is greater than or equal to the
+ given version. If `accept_dev` is True, it will also accept development versions (e.g. 2.7.0.dev20250320 matches
+ 2.7.0).
+ """
+ if not _is_package_available("torch"):
+ return False
+
+ if accept_dev:
+ return version.parse(version.parse(importlib.metadata.version("torch")).base_version) >= version.parse(
+ library_version
+ )
+ else:
+ return version.parse(importlib.metadata.version("torch")) >= version.parse(library_version)
+
+
+@lru_cache
+def is_torch_less_or_equal(library_version: str, accept_dev: bool = False) -> bool:
+ """
+ Accepts a library version and returns True if the current version of the library is less than or equal to the
+ given version. If `accept_dev` is True, it will also accept development versions (e.g. 2.7.0.dev20250320 matches
+ 2.7.0).
+ """
+ if not _is_package_available("torch"):
+ return False
+
+ if accept_dev:
+ return version.parse(version.parse(importlib.metadata.version("torch")).base_version) <= version.parse(
+ library_version
+ )
+ else:
+ return version.parse(importlib.metadata.version("torch")) <= version.parse(library_version)
+
+
+@lru_cache
+def is_huggingface_hub_greater_or_equal(library_version: str, accept_dev: bool = False) -> bool:
+ if not _is_package_available("huggingface_hub"):
+ return False
+
+ if accept_dev:
+ return version.parse(
+ version.parse(importlib.metadata.version("huggingface_hub")).base_version
+ ) >= version.parse(library_version)
+ else:
+ return version.parse(importlib.metadata.version("huggingface_hub")) >= version.parse(library_version)
+
+
+@lru_cache
+def is_quanto_greater(library_version: str, accept_dev: bool = False) -> bool:
+ """
+ Accepts a library version and returns True if the current version of the library is greater than or equal to the
+ given version. If `accept_dev` is True, it will also accept development versions (e.g. 2.7.0.dev20250320 matches
+ 2.7.0).
+ """
+ if not _is_package_available("optimum.quanto"):
+ return False
+
+ if accept_dev:
+ return version.parse(version.parse(importlib.metadata.version("optimum-quanto")).base_version) > version.parse(
+ library_version
+ )
+ else:
+ return version.parse(importlib.metadata.version("optimum-quanto")) > version.parse(library_version)
+
+
+def is_torchdistx_available():
+ return _torchdistx_available
+
+
+def is_faiss_available() -> bool:
+ return _faiss_available
+
+
+def is_scipy_available() -> Union[tuple[bool, str], bool]:
+ return _scipy_available
+
+
+def is_sklearn_available() -> Union[tuple[bool, str], bool]:
+ return _sklearn_available
+
+
+def is_sentencepiece_available() -> Union[tuple[bool, str], bool]:
+ return _sentencepiece_available
+
+
+def is_seqio_available() -> Union[tuple[bool, str], bool]:
+ return _is_seqio_available
+
+
+def is_gguf_available(min_version: str = GGUF_MIN_VERSION) -> bool:
+ return _is_gguf_available and version.parse(_gguf_version) >= version.parse(min_version)
+
+
+def is_protobuf_available() -> bool:
+ if importlib.util.find_spec("google") is None:
+ return False
+ return importlib.util.find_spec("google.protobuf") is not None
+
+
+def is_fsdp_available(min_version: str = FSDP_MIN_VERSION) -> bool:
+ return is_torch_available() and version.parse(_torch_version) >= version.parse(min_version)
+
+
+def is_optimum_available() -> Union[tuple[bool, str], bool]:
+ return _optimum_available
+
+
+def is_auto_awq_available() -> bool:
+ return _auto_awq_available
+
+
+def is_auto_round_available(min_version: str = AUTOROUND_MIN_VERSION) -> bool:
+ return _auto_round_available and version.parse(_auto_round_version) >= version.parse(min_version)
+
+
+def is_optimum_quanto_available():
+ # `importlib.metadata.version` doesn't work with `optimum.quanto`, need to put `optimum_quanto`
+ return _is_optimum_quanto_available
+
+
+def is_quark_available() -> Union[tuple[bool, str], bool]:
+ return _quark_available
+
+
+def is_fp_quant_available() -> bool:
+ return _fp_quant_available and version.parse(_fp_quant_version) >= version.parse("0.1.6")
+
+
+def is_qutlass_available() -> Union[tuple[bool, str], bool]:
+ return _qutlass_available
+
+
+def is_compressed_tensors_available() -> bool:
+ return _compressed_tensors_available
+
+
+def is_auto_gptq_available() -> Union[tuple[bool, str], bool]:
+ return _auto_gptq_available
+
+
+def is_gptqmodel_available() -> Union[tuple[bool, str], bool]:
+ return _gptqmodel_available
+
+
+def is_eetq_available() -> Union[tuple[bool, str], bool]:
+ return _eetq_available
+
+
+def is_fbgemm_gpu_available() -> Union[tuple[bool, str], bool]:
+ return _fbgemm_gpu_available
+
+
+def is_levenshtein_available() -> Union[tuple[bool, str], bool]:
+ return _levenshtein_available
+
+
+def is_optimum_neuron_available() -> Union[tuple[bool, str], bool]:
+ return _optimum_available and _is_package_available("optimum.neuron")
+
+
+def is_safetensors_available() -> Union[tuple[bool, str], bool]:
+ return _safetensors_available
+
+
+def is_tokenizers_available() -> Union[tuple[bool, str], bool]:
+ return _tokenizers_available
+
+
+@lru_cache
+def is_vision_available() -> bool:
+ _pil_available = importlib.util.find_spec("PIL") is not None
+ if _pil_available:
+ try:
+ package_version = importlib.metadata.version("Pillow")
+ except importlib.metadata.PackageNotFoundError:
+ try:
+ package_version = importlib.metadata.version("Pillow-SIMD")
+ except importlib.metadata.PackageNotFoundError:
+ return False
+ logger.debug(f"Detected PIL version {package_version}")
+ return _pil_available
+
+
+def is_pytesseract_available() -> Union[tuple[bool, str], bool]:
+ return _pytesseract_available
+
+
+def is_pytest_available() -> Union[tuple[bool, str], bool]:
+ return _pytest_available
+
+
+def is_spacy_available() -> Union[tuple[bool, str], bool]:
+ return _spacy_available
+
+
+def is_tensorflow_text_available() -> Union[tuple[bool, str], bool]:
+ return is_tf_available() and _tensorflow_text_available
+
+
+def is_keras_nlp_available() -> Union[tuple[bool, str], bool]:
+ return is_tensorflow_text_available() and _keras_nlp_available
+
+
+def is_in_notebook() -> bool:
+ try:
+ # Check if we are running inside Marimo
+ if "marimo" in sys.modules:
+ return True
+ # Test adapted from tqdm.autonotebook: https://github.com/tqdm/tqdm/blob/master/tqdm/autonotebook.py
+ get_ipython = sys.modules["IPython"].get_ipython
+ if "IPKernelApp" not in get_ipython().config:
+ raise ImportError("console")
+ # Removed the lines to include VSCode
+ if "DATABRICKS_RUNTIME_VERSION" in os.environ and os.environ["DATABRICKS_RUNTIME_VERSION"] < "11.0":
+ # Databricks Runtime 11.0 and above uses IPython kernel by default so it should be compatible with Jupyter notebook
+ # https://docs.microsoft.com/en-us/azure/databricks/notebooks/ipython-kernel
+ raise ImportError("databricks")
+
+ return importlib.util.find_spec("IPython") is not None
+ except (AttributeError, ImportError, KeyError):
+ return False
+
+
+def is_pytorch_quantization_available() -> Union[tuple[bool, str], bool]:
+ return _pytorch_quantization_available
+
+
+def is_tensorflow_probability_available() -> Union[tuple[bool, str], bool]:
+ return _tensorflow_probability_available
+
+
+def is_pandas_available() -> Union[tuple[bool, str], bool]:
+ return _pandas_available
+
+
+def is_sagemaker_dp_enabled() -> bool:
+ # Get the sagemaker specific env variable.
+ sagemaker_params = os.getenv("SM_FRAMEWORK_PARAMS", "{}")
+ try:
+ # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
+ sagemaker_params = json.loads(sagemaker_params)
+ if not sagemaker_params.get("sagemaker_distributed_dataparallel_enabled", False):
+ return False
+ except json.JSONDecodeError:
+ return False
+ # Lastly, check if the `smdistributed` module is present.
+ return _smdistributed_available
+
+
+def is_sagemaker_mp_enabled() -> bool:
+ # Get the sagemaker specific mp parameters from smp_options variable.
+ smp_options = os.getenv("SM_HP_MP_PARAMETERS", "{}")
+ try:
+ # Parse it and check the field "partitions" is included, it is required for model parallel.
+ smp_options = json.loads(smp_options)
+ if "partitions" not in smp_options:
+ return False
+ except json.JSONDecodeError:
+ return False
+
+ # Get the sagemaker specific framework parameters from mpi_options variable.
+ mpi_options = os.getenv("SM_FRAMEWORK_PARAMS", "{}")
+ try:
+ # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
+ mpi_options = json.loads(mpi_options)
+ if not mpi_options.get("sagemaker_mpi_enabled", False):
+ return False
+ except json.JSONDecodeError:
+ return False
+ # Lastly, check if the `smdistributed` module is present.
+ return _smdistributed_available
+
+
+def is_training_run_on_sagemaker() -> bool:
+ return "SAGEMAKER_JOB_NAME" in os.environ
+
+
+def is_soundfile_available() -> Union[tuple[bool, str], bool]:
+ return _soundfile_available
+
+
+def is_timm_available() -> Union[tuple[bool, str], bool]:
+ return _timm_available
+
+
+def is_natten_available() -> Union[tuple[bool, str], bool]:
+ return _natten_available
+
+
+def is_nltk_available() -> Union[tuple[bool, str], bool]:
+ return _nltk_available
+
+
+def is_torchaudio_available() -> Union[tuple[bool, str], bool]:
+ return _torchaudio_available
+
+
+def is_torchao_available(min_version: str = TORCHAO_MIN_VERSION) -> bool:
+ return _torchao_available and version.parse(_torchao_version) >= version.parse(min_version)
+
+
+def is_speech_available() -> Union[tuple[bool, str], bool]:
+ # For now this depends on torchaudio but the exact dependency might evolve in the future.
+ return _torchaudio_available
+
+
+def is_spqr_available() -> Union[tuple[bool, str], bool]:
+ return _spqr_available
+
+
+def is_phonemizer_available() -> Union[tuple[bool, str], bool]:
+ return _phonemizer_available
+
+
+def is_uroman_available() -> Union[tuple[bool, str], bool]:
+ return _uroman_available
+
+
+def torch_only_method(fn: Callable) -> Callable:
+ def wrapper(*args, **kwargs):
+ if not _torch_available:
+ raise ImportError(
+ "You need to install pytorch to use this method or class, "
+ "or activate it with environment variables USE_TORCH=1 and USE_TF=0."
+ )
+ else:
+ return fn(*args, **kwargs)
+
+ return wrapper
+
+
+def is_ccl_available() -> bool:
+ return _is_ccl_available
+
+
+def is_sudachi_available() -> bool:
+ return _sudachipy_available
+
+
+def get_sudachi_version() -> bool:
+ return _sudachipy_version
+
+
+def is_sudachi_projection_available() -> bool:
+ if not is_sudachi_available():
+ return False
+
+ # NOTE: We require sudachipy>=0.6.8 to use projection option in sudachi_kwargs for the constructor of BertJapaneseTokenizer.
+ # - `projection` option is not supported in sudachipy<0.6.8, see https://github.com/WorksApplications/sudachi.rs/issues/230
+ return version.parse(_sudachipy_version) >= version.parse("0.6.8")
+
+
+def is_jumanpp_available() -> bool:
+ return (importlib.util.find_spec("rhoknp") is not None) and (shutil.which("jumanpp") is not None)
+
+
+def is_cython_available() -> bool:
+ return importlib.util.find_spec("pyximport") is not None
+
+
+def is_jieba_available() -> Union[tuple[bool, str], bool]:
+ return _jieba_available
+
+
+def is_jinja_available() -> Union[tuple[bool, str], bool]:
+ return _jinja_available
+
+
+def is_mlx_available() -> Union[tuple[bool, str], bool]:
+ return _mlx_available
+
+
+def is_num2words_available() -> Union[tuple[bool, str], bool]:
+ return _num2words_available
+
+
+def is_tiktoken_available() -> Union[tuple[bool, str], bool]:
+ return _tiktoken_available and _blobfile_available
+
+
+def is_liger_kernel_available() -> bool:
+ if not _liger_kernel_available:
+ return False
+
+ return version.parse(importlib.metadata.version("liger_kernel")) >= version.parse("0.3.0")
+
+
+def is_rich_available() -> Union[tuple[bool, str], bool]:
+ return _rich_available
+
+
+def is_matplotlib_available() -> Union[tuple[bool, str], bool]:
+ return _matplotlib_available
+
+
+def is_mistral_common_available() -> Union[tuple[bool, str], bool]:
+ return _mistral_common_available
+
+
+def check_torch_load_is_safe() -> None:
+ if not is_torch_greater_or_equal("2.6"):
+ raise ValueError(
+ "Due to a serious vulnerability issue in `torch.load`, even with `weights_only=True`, we now require users "
+ "to upgrade torch to at least v2.6 in order to use the function. This version restriction does not apply "
+ "when loading files with safetensors."
+ "\nSee the vulnerability report here https://nvd.nist.gov/vuln/detail/CVE-2025-32434"
+ )
+
+
+# docstyle-ignore
+AV_IMPORT_ERROR = """
+{0} requires the PyAv library but it was not found in your environment. You can install it with:
+```
+pip install av
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+YT_DLP_IMPORT_ERROR = """
+{0} requires the YT-DLP library but it was not found in your environment. You can install it with:
+```
+pip install yt-dlp
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+DECORD_IMPORT_ERROR = """
+{0} requires the PyAv library but it was not found in your environment. You can install it with:
+```
+pip install decord
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+TORCHCODEC_IMPORT_ERROR = """
+{0} requires the TorchCodec (https://github.com/pytorch/torchcodec) library, but it was not found in your environment. You can install it with:
+```
+pip install torchcodec
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+CV2_IMPORT_ERROR = """
+{0} requires the OpenCV library but it was not found in your environment. You can install it with:
+```
+pip install opencv-python
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+DATASETS_IMPORT_ERROR = """
+{0} requires the 🤗 Datasets library but it was not found in your environment. You can install it with:
+```
+pip install datasets
+```
+In a notebook or a colab, you can install it by executing a cell with
+```
+!pip install datasets
+```
+then restarting your kernel.
+
+Note that if you have a local folder named `datasets` or a local python file named `datasets.py` in your current
+working directory, python may try to import this instead of the 🤗 Datasets library. You should rename this folder or
+that python file if that's the case. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+TOKENIZERS_IMPORT_ERROR = """
+{0} requires the 🤗 Tokenizers library but it was not found in your environment. You can install it with:
+```
+pip install tokenizers
+```
+In a notebook or a colab, you can install it by executing a cell with
+```
+!pip install tokenizers
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+SENTENCEPIECE_IMPORT_ERROR = """
+{0} requires the SentencePiece library but it was not found in your environment. Check out the instructions on the
+installation page of its repo: https://github.com/google/sentencepiece#installation and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+PROTOBUF_IMPORT_ERROR = """
+{0} requires the protobuf library but it was not found in your environment. Check out the instructions on the
+installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+FAISS_IMPORT_ERROR = """
+{0} requires the faiss library but it was not found in your environment. Check out the instructions on the
+installation page of its repo: https://github.com/facebookresearch/faiss/blob/master/INSTALL.md and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+PYTORCH_IMPORT_ERROR = """
+{0} requires the PyTorch library but it was not found in your environment. Check out the instructions on the
+installation page: https://pytorch.org/get-started/locally/ and follow the ones that match your environment.
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+TORCHVISION_IMPORT_ERROR = """
+{0} requires the Torchvision library but it was not found in your environment. Check out the instructions on the
+installation page: https://pytorch.org/get-started/locally/ and follow the ones that match your environment.
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PYTORCH_IMPORT_ERROR_WITH_TF = """
+{0} requires the PyTorch library but it was not found in your environment.
+However, we were able to find a TensorFlow installation. TensorFlow classes begin
+with "TF", but are otherwise identically named to our PyTorch classes. This
+means that the TF equivalent of the class you tried to import would be "TF{0}".
+If you want to use TensorFlow, please use TF classes instead!
+
+If you really do want to use PyTorch please go to
+https://pytorch.org/get-started/locally/ and follow the instructions that
+match your environment.
+"""
+
+# docstyle-ignore
+TF_IMPORT_ERROR_WITH_PYTORCH = """
+{0} requires the TensorFlow library but it was not found in your environment.
+However, we were able to find a PyTorch installation. PyTorch classes do not begin
+with "TF", but are otherwise identically named to our TF classes.
+If you want to use PyTorch, please use those classes instead!
+
+If you really do want to use TensorFlow, please follow the instructions on the
+installation page https://www.tensorflow.org/install that match your environment.
+"""
+
+# docstyle-ignore
+BS4_IMPORT_ERROR = """
+{0} requires the Beautiful Soup library but it was not found in your environment. You can install it with pip:
+`pip install beautifulsoup4`. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+SKLEARN_IMPORT_ERROR = """
+{0} requires the scikit-learn library but it was not found in your environment. You can install it with:
+```
+pip install -U scikit-learn
+```
+In a notebook or a colab, you can install it by executing a cell with
+```
+!pip install -U scikit-learn
+```
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+TENSORFLOW_IMPORT_ERROR = """
+{0} requires the TensorFlow library but it was not found in your environment. Check out the instructions on the
+installation page: https://www.tensorflow.org/install and follow the ones that match your environment.
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+DETECTRON2_IMPORT_ERROR = """
+{0} requires the detectron2 library but it was not found in your environment. Check out the instructions on the
+installation page: https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+FLAX_IMPORT_ERROR = """
+{0} requires the FLAX library but it was not found in your environment. Check out the instructions on the
+installation page: https://github.com/google/flax and follow the ones that match your environment.
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+FTFY_IMPORT_ERROR = """
+{0} requires the ftfy library but it was not found in your environment. Check out the instructions on the
+installation section: https://github.com/rspeer/python-ftfy/tree/master#installing and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+"""
+
+LEVENSHTEIN_IMPORT_ERROR = """
+{0} requires the python-Levenshtein library but it was not found in your environment. You can install it with pip: `pip
+install python-Levenshtein`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+G2P_EN_IMPORT_ERROR = """
+{0} requires the g2p-en library but it was not found in your environment. You can install it with pip:
+`pip install g2p-en`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PYTORCH_QUANTIZATION_IMPORT_ERROR = """
+{0} requires the pytorch-quantization library but it was not found in your environment. You can install it with pip:
+`pip install pytorch-quantization --extra-index-url https://pypi.ngc.nvidia.com`
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+TENSORFLOW_PROBABILITY_IMPORT_ERROR = """
+{0} requires the tensorflow_probability library but it was not found in your environment. You can install it with pip as
+explained here: https://github.com/tensorflow/probability. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+TENSORFLOW_TEXT_IMPORT_ERROR = """
+{0} requires the tensorflow_text library but it was not found in your environment. You can install it with pip as
+explained here: https://www.tensorflow.org/text/guide/tf_text_intro.
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+TORCHAUDIO_IMPORT_ERROR = """
+{0} requires the torchaudio library but it was not found in your environment. Please install it and restart your
+runtime.
+"""
+
+# docstyle-ignore
+PANDAS_IMPORT_ERROR = """
+{0} requires the pandas library but it was not found in your environment. You can install it with pip as
+explained here: https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html.
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+PHONEMIZER_IMPORT_ERROR = """
+{0} requires the phonemizer library but it was not found in your environment. You can install it with pip:
+`pip install phonemizer`. Please note that you may need to restart your runtime after installation.
+"""
+# docstyle-ignore
+UROMAN_IMPORT_ERROR = """
+{0} requires the uroman library but it was not found in your environment. You can install it with pip:
+`pip install uroman`. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+SACREMOSES_IMPORT_ERROR = """
+{0} requires the sacremoses library but it was not found in your environment. You can install it with pip:
+`pip install sacremoses`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+SCIPY_IMPORT_ERROR = """
+{0} requires the scipy library but it was not found in your environment. You can install it with pip:
+`pip install scipy`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+KERAS_NLP_IMPORT_ERROR = """
+{0} requires the keras_nlp library but it was not found in your environment. You can install it with pip.
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+SPEECH_IMPORT_ERROR = """
+{0} requires the torchaudio library but it was not found in your environment. You can install it with pip:
+`pip install torchaudio`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+TIMM_IMPORT_ERROR = """
+{0} requires the timm library but it was not found in your environment. You can install it with pip:
+`pip install timm`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+NATTEN_IMPORT_ERROR = """
+{0} requires the natten library but it was not found in your environment. You can install it by referring to:
+shi-labs.com/natten . You can also install it with pip (may take longer to build):
+`pip install natten`. Please note that you may need to restart your runtime after installation.
+"""
+
+NUMEXPR_IMPORT_ERROR = """
+{0} requires the numexpr library but it was not found in your environment. You can install it by referring to:
+https://numexpr.readthedocs.io/en/latest/index.html.
+"""
+
+
+# docstyle-ignore
+NLTK_IMPORT_ERROR = """
+{0} requires the NLTK library but it was not found in your environment. You can install it by referring to:
+https://www.nltk.org/install.html. Please note that you may need to restart your runtime after installation.
+"""
+
+
+# docstyle-ignore
+VISION_IMPORT_ERROR = """
+{0} requires the PIL library but it was not found in your environment. You can install it with pip:
+`pip install pillow`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PYDANTIC_IMPORT_ERROR = """
+{0} requires the pydantic library but it was not found in your environment. You can install it with pip:
+`pip install pydantic`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+FASTAPI_IMPORT_ERROR = """
+{0} requires the fastapi library but it was not found in your environment. You can install it with pip:
+`pip install fastapi`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+UVICORN_IMPORT_ERROR = """
+{0} requires the uvicorn library but it was not found in your environment. You can install it with pip:
+`pip install uvicorn`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+OPENAI_IMPORT_ERROR = """
+{0} requires the openai library but it was not found in your environment. You can install it with pip:
+`pip install openai`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PYTESSERACT_IMPORT_ERROR = """
+{0} requires the PyTesseract library but it was not found in your environment. You can install it with pip:
+`pip install pytesseract`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PYCTCDECODE_IMPORT_ERROR = """
+{0} requires the pyctcdecode library but it was not found in your environment. You can install it with pip:
+`pip install pyctcdecode`. Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+ACCELERATE_IMPORT_ERROR = """
+{0} requires the accelerate library >= {ACCELERATE_MIN_VERSION} it was not found in your environment.
+You can install or update it with pip: `pip install --upgrade accelerate`. Please note that you may need to restart your
+runtime after installation.
+"""
+
+# docstyle-ignore
+CCL_IMPORT_ERROR = """
+{0} requires the torch ccl library but it was not found in your environment. You can install it with pip:
+`pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable`
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+ESSENTIA_IMPORT_ERROR = """
+{0} requires essentia library. But that was not found in your environment. You can install them with pip:
+`pip install essentia==2.1b6.dev1034`
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+LIBROSA_IMPORT_ERROR = """
+{0} requires the librosa library. But that was not found in your environment. You can install them with pip:
+`pip install librosa`
+Please note that you may need to restart your runtime after installation.
+"""
+
+# docstyle-ignore
+PRETTY_MIDI_IMPORT_ERROR = """
+{0} requires the pretty_midi library. But that was not found in your environment. You can install them with pip:
+`pip install pretty_midi`
+Please note that you may need to restart your runtime after installation.
+"""
+
+
+CYTHON_IMPORT_ERROR = """
+{0} requires the Cython library but it was not found in your environment. You can install it with pip: `pip install
+Cython`. Please note that you may need to restart your runtime after installation.
+"""
+
+JIEBA_IMPORT_ERROR = """
+{0} requires the jieba library but it was not found in your environment. You can install it with pip: `pip install
+jieba`. Please note that you may need to restart your runtime after installation.
+"""
+
+PEFT_IMPORT_ERROR = """
+{0} requires the peft library but it was not found in your environment. You can install it with pip: `pip install
+peft`. Please note that you may need to restart your runtime after installation.
+"""
+
+JINJA_IMPORT_ERROR = """
+{0} requires the jinja library but it was not found in your environment. You can install it with pip: `pip install
+jinja2`. Please note that you may need to restart your runtime after installation.
+"""
+
+RICH_IMPORT_ERROR = """
+{0} requires the rich library but it was not found in your environment. You can install it with pip: `pip install
+rich`. Please note that you may need to restart your runtime after installation.
+"""
+
+MISTRAL_COMMON_IMPORT_ERROR = """
+{0} requires the mistral-common library but it was not found in your environment. You can install it with pip: `pip install mistral-common`. Please note that you may need to restart your runtime after installation.
+"""
+
+
+BACKENDS_MAPPING = OrderedDict(
+ [
+ ("av", (is_av_available, AV_IMPORT_ERROR)),
+ ("bs4", (is_bs4_available, BS4_IMPORT_ERROR)),
+ ("cv2", (is_cv2_available, CV2_IMPORT_ERROR)),
+ ("datasets", (is_datasets_available, DATASETS_IMPORT_ERROR)),
+ ("decord", (is_decord_available, DECORD_IMPORT_ERROR)),
+ ("detectron2", (is_detectron2_available, DETECTRON2_IMPORT_ERROR)),
+ ("essentia", (is_essentia_available, ESSENTIA_IMPORT_ERROR)),
+ ("faiss", (is_faiss_available, FAISS_IMPORT_ERROR)),
+ ("flax", (is_flax_available, FLAX_IMPORT_ERROR)),
+ ("ftfy", (is_ftfy_available, FTFY_IMPORT_ERROR)),
+ ("g2p_en", (is_g2p_en_available, G2P_EN_IMPORT_ERROR)),
+ ("pandas", (is_pandas_available, PANDAS_IMPORT_ERROR)),
+ ("phonemizer", (is_phonemizer_available, PHONEMIZER_IMPORT_ERROR)),
+ ("uroman", (is_uroman_available, UROMAN_IMPORT_ERROR)),
+ ("pretty_midi", (is_pretty_midi_available, PRETTY_MIDI_IMPORT_ERROR)),
+ ("levenshtein", (is_levenshtein_available, LEVENSHTEIN_IMPORT_ERROR)),
+ ("librosa", (is_librosa_available, LIBROSA_IMPORT_ERROR)),
+ ("protobuf", (is_protobuf_available, PROTOBUF_IMPORT_ERROR)),
+ ("pyctcdecode", (is_pyctcdecode_available, PYCTCDECODE_IMPORT_ERROR)),
+ ("pytesseract", (is_pytesseract_available, PYTESSERACT_IMPORT_ERROR)),
+ ("sacremoses", (is_sacremoses_available, SACREMOSES_IMPORT_ERROR)),
+ ("pytorch_quantization", (is_pytorch_quantization_available, PYTORCH_QUANTIZATION_IMPORT_ERROR)),
+ ("sentencepiece", (is_sentencepiece_available, SENTENCEPIECE_IMPORT_ERROR)),
+ ("sklearn", (is_sklearn_available, SKLEARN_IMPORT_ERROR)),
+ ("speech", (is_speech_available, SPEECH_IMPORT_ERROR)),
+ ("tensorflow_probability", (is_tensorflow_probability_available, TENSORFLOW_PROBABILITY_IMPORT_ERROR)),
+ ("tf", (is_tf_available, TENSORFLOW_IMPORT_ERROR)),
+ ("tensorflow_text", (is_tensorflow_text_available, TENSORFLOW_TEXT_IMPORT_ERROR)),
+ ("timm", (is_timm_available, TIMM_IMPORT_ERROR)),
+ ("torchaudio", (is_torchaudio_available, TORCHAUDIO_IMPORT_ERROR)),
+ ("natten", (is_natten_available, NATTEN_IMPORT_ERROR)),
+ ("nltk", (is_nltk_available, NLTK_IMPORT_ERROR)),
+ ("tokenizers", (is_tokenizers_available, TOKENIZERS_IMPORT_ERROR)),
+ ("torch", (is_torch_available, PYTORCH_IMPORT_ERROR)),
+ ("torchvision", (is_torchvision_available, TORCHVISION_IMPORT_ERROR)),
+ ("torchcodec", (is_torchcodec_available, TORCHCODEC_IMPORT_ERROR)),
+ ("vision", (is_vision_available, VISION_IMPORT_ERROR)),
+ ("scipy", (is_scipy_available, SCIPY_IMPORT_ERROR)),
+ ("accelerate", (is_accelerate_available, ACCELERATE_IMPORT_ERROR)),
+ ("oneccl_bind_pt", (is_ccl_available, CCL_IMPORT_ERROR)),
+ ("cython", (is_cython_available, CYTHON_IMPORT_ERROR)),
+ ("jieba", (is_jieba_available, JIEBA_IMPORT_ERROR)),
+ ("peft", (is_peft_available, PEFT_IMPORT_ERROR)),
+ ("jinja", (is_jinja_available, JINJA_IMPORT_ERROR)),
+ ("yt_dlp", (is_yt_dlp_available, YT_DLP_IMPORT_ERROR)),
+ ("rich", (is_rich_available, RICH_IMPORT_ERROR)),
+ ("keras_nlp", (is_keras_nlp_available, KERAS_NLP_IMPORT_ERROR)),
+ ("pydantic", (is_pydantic_available, PYDANTIC_IMPORT_ERROR)),
+ ("fastapi", (is_fastapi_available, FASTAPI_IMPORT_ERROR)),
+ ("uvicorn", (is_uvicorn_available, UVICORN_IMPORT_ERROR)),
+ ("openai", (is_openai_available, OPENAI_IMPORT_ERROR)),
+ ("mistral-common", (is_mistral_common_available, MISTRAL_COMMON_IMPORT_ERROR)),
+ ]
+)
+
+
+def requires_backends(obj, backends):
+ if not isinstance(backends, (list, tuple)):
+ backends = [backends]
+
+ name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
+
+ # Raise an error for users who might not realize that classes without "TF" are torch-only
+ if "torch" in backends and "tf" not in backends and not is_torch_available() and is_tf_available():
+ raise ImportError(PYTORCH_IMPORT_ERROR_WITH_TF.format(name))
+
+ # Raise the inverse error for PyTorch users trying to load TF classes
+ if "tf" in backends and "torch" not in backends and is_torch_available() and not is_tf_available():
+ raise ImportError(TF_IMPORT_ERROR_WITH_PYTORCH.format(name))
+
+ failed = []
+ for backend in backends:
+ if isinstance(backend, Backend):
+ available, msg = backend.is_satisfied, backend.error_message
+ else:
+ available, msg = BACKENDS_MAPPING[backend]
+
+ if not available():
+ failed.append(msg.format(name))
+
+ if failed:
+ raise ImportError("".join(failed))
+
+
+class DummyObject(type):
+ """
+ Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by
+ `requires_backend` each time a user tries to access any method of that class.
+ """
+
+ is_dummy = True
+
+ def __getattribute__(cls, key):
+ if (key.startswith("_") and key != "_from_config") or key == "is_dummy" or key == "mro" or key == "call":
+ return super().__getattribute__(key)
+ requires_backends(cls, cls._backends)
+
+
+def is_torch_fx_proxy(x):
+ if is_torch_fx_available():
+ import torch.fx
+
+ return isinstance(x, torch.fx.Proxy)
+ return False
+
+
+BACKENDS_T = frozenset[str]
+IMPORT_STRUCTURE_T = dict[BACKENDS_T, dict[str, set[str]]]
+
+
+class _LazyModule(ModuleType):
+ """
+ Module class that surfaces all objects but only performs associated imports when the objects are requested.
+ """
+
+ # Very heavily inspired by optuna.integration._IntegrationModule
+ # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py
+ def __init__(
+ self,
+ name: str,
+ module_file: str,
+ import_structure: IMPORT_STRUCTURE_T,
+ module_spec: Optional[importlib.machinery.ModuleSpec] = None,
+ extra_objects: Optional[dict[str, object]] = None,
+ explicit_import_shortcut: Optional[dict[str, list[str]]] = None,
+ ):
+ super().__init__(name)
+
+ self._object_missing_backend = {}
+ self._explicit_import_shortcut = explicit_import_shortcut if explicit_import_shortcut else {}
+
+ if any(isinstance(key, frozenset) for key in import_structure):
+ self._modules = set()
+ self._class_to_module = {}
+ self.__all__ = []
+
+ _import_structure = {}
+
+ for backends, module in import_structure.items():
+ missing_backends = []
+
+ # This ensures that if a module is importable, then all other keys of the module are importable.
+ # As an example, in module.keys() we might have the following:
+ #
+ # dict_keys(['models.nllb_moe.configuration_nllb_moe', 'models.sew_d.configuration_sew_d'])
+ #
+ # with this, we don't only want to be able to import these explicitly, we want to be able to import
+ # every intermediate module as well. Therefore, this is what is returned:
+ #
+ # {
+ # 'models.nllb_moe.configuration_nllb_moe',
+ # 'models.sew_d.configuration_sew_d',
+ # 'models',
+ # 'models.sew_d', 'models.nllb_moe'
+ # }
+
+ module_keys = set(
+ chain(*[[k.rsplit(".", i)[0] for i in range(k.count(".") + 1)] for k in list(module.keys())])
+ )
+
+ for backend in backends:
+ if backend in BACKENDS_MAPPING:
+ callable, _ = BACKENDS_MAPPING[backend]
+ else:
+ if any(key in backend for key in ["=", "<", ">"]):
+ backend = Backend(backend)
+ callable = backend.is_satisfied
+ else:
+ raise ValueError(
+ f"Backend should be defined in the BACKENDS_MAPPING. Offending backend: {backend}"
+ )
+
+ try:
+ if not callable():
+ missing_backends.append(backend)
+ except (importlib.metadata.PackageNotFoundError, ModuleNotFoundError, RuntimeError):
+ missing_backends.append(backend)
+
+ self._modules = self._modules.union(module_keys)
+
+ for key, values in module.items():
+ if missing_backends:
+ self._object_missing_backend[key] = missing_backends
+
+ for value in values:
+ self._class_to_module[value] = key
+ if missing_backends:
+ self._object_missing_backend[value] = missing_backends
+ _import_structure.setdefault(key, []).extend(values)
+
+ # Needed for autocompletion in an IDE
+ self.__all__.extend(module_keys | set(chain(*module.values())))
+
+ self.__file__ = module_file
+ self.__spec__ = module_spec
+ self.__path__ = [os.path.dirname(module_file)]
+ self._objects = {} if extra_objects is None else extra_objects
+ self._name = name
+ self._import_structure = _import_structure
+
+ # This can be removed once every exportable object has a `require()` require.
+ else:
+ self._modules = set(import_structure.keys())
+ self._class_to_module = {}
+ for key, values in import_structure.items():
+ for value in values:
+ self._class_to_module[value] = key
+ # Needed for autocompletion in an IDE
+ self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values()))
+ self.__file__ = module_file
+ self.__spec__ = module_spec
+ self.__path__ = [os.path.dirname(module_file)]
+ self._objects = {} if extra_objects is None else extra_objects
+ self._name = name
+ self._import_structure = import_structure
+
+ # Needed for autocompletion in an IDE
+ def __dir__(self):
+ result = super().__dir__()
+ # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether
+ # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir.
+ for attr in self.__all__:
+ if attr not in result:
+ result.append(attr)
+ return result
+
+ def __getattr__(self, name: str) -> Any:
+ if name in self._objects:
+ return self._objects[name]
+ if name in self._object_missing_backend:
+ missing_backends = self._object_missing_backend[name]
+
+ class Placeholder(metaclass=DummyObject):
+ _backends = missing_backends
+
+ def __init__(self, *args, **kwargs):
+ requires_backends(self, missing_backends)
+
+ def call(self, *args, **kwargs):
+ pass
+
+ Placeholder.__name__ = name
+
+ if name not in self._class_to_module:
+ module_name = f"transformers.{name}"
+ else:
+ module_name = self._class_to_module[name]
+ if not module_name.startswith("transformers."):
+ module_name = f"transformers.{module_name}"
+
+ Placeholder.__module__ = module_name
+
+ value = Placeholder
+ elif name in self._class_to_module:
+ try:
+ module = self._get_module(self._class_to_module[name])
+ value = getattr(module, name)
+ except (ModuleNotFoundError, RuntimeError) as e:
+ raise ModuleNotFoundError(
+ f"Could not import module '{name}'. Are this object's requirements defined correctly?"
+ ) from e
+
+ elif name in self._modules:
+ try:
+ value = self._get_module(name)
+ except (ModuleNotFoundError, RuntimeError) as e:
+ raise ModuleNotFoundError(
+ f"Could not import module '{name}'. Are this object's requirements defined correctly?"
+ ) from e
+ else:
+ value = None
+ for key, values in self._explicit_import_shortcut.items():
+ if name in values:
+ value = self._get_module(key)
+
+ if value is None:
+ raise AttributeError(f"module {self.__name__} has no attribute {name}")
+
+ setattr(self, name, value)
+ return value
+
+ def _get_module(self, module_name: str):
+ try:
+ return importlib.import_module("." + module_name, self.__name__)
+ except Exception as e:
+ raise e
+
+ def __reduce__(self):
+ return (self.__class__, (self._name, self.__file__, self._import_structure))
+
+
+class OptionalDependencyNotAvailable(BaseException):
+ """Internally used error class for signalling an optional dependency was not found."""
+
+
+def direct_transformers_import(path: str, file="__init__.py") -> ModuleType:
+ """Imports transformers directly
+
+ Args:
+ path (`str`): The path to the source file
+ file (`str`, *optional*): The file to join with the path. Defaults to "__init__.py".
+
+ Returns:
+ `ModuleType`: The resulting imported module
+ """
+ name = "transformers"
+ location = os.path.join(path, file)
+ spec = importlib.util.spec_from_file_location(name, location, submodule_search_locations=[path])
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module = sys.modules[name]
+ return module
+
+
+class VersionComparison(Enum):
+ EQUAL = operator.eq
+ NOT_EQUAL = operator.ne
+ GREATER_THAN = operator.gt
+ LESS_THAN = operator.lt
+ GREATER_THAN_OR_EQUAL = operator.ge
+ LESS_THAN_OR_EQUAL = operator.le
+
+ @staticmethod
+ def from_string(version_string: str) -> "VersionComparison":
+ string_to_operator = {
+ "=": VersionComparison.EQUAL.value,
+ "==": VersionComparison.EQUAL.value,
+ "!=": VersionComparison.NOT_EQUAL.value,
+ ">": VersionComparison.GREATER_THAN.value,
+ "<": VersionComparison.LESS_THAN.value,
+ ">=": VersionComparison.GREATER_THAN_OR_EQUAL.value,
+ "<=": VersionComparison.LESS_THAN_OR_EQUAL.value,
+ }
+
+ return string_to_operator[version_string]
+
+
+@lru_cache
+def split_package_version(package_version_str) -> tuple[str, str, str]:
+ pattern = r"([a-zA-Z0-9_-]+)([!<>=~]+)([0-9.]+)"
+ match = re.match(pattern, package_version_str)
+ if match:
+ return (match.group(1), match.group(2), match.group(3))
+ else:
+ raise ValueError(f"Invalid package version string: {package_version_str}")
+
+
+class Backend:
+ def __init__(self, backend_requirement: str):
+ self.package_name, self.version_comparison, self.version = split_package_version(backend_requirement)
+
+ if self.package_name not in BACKENDS_MAPPING:
+ raise ValueError(
+ f"Backends should be defined in the BACKENDS_MAPPING. Offending backend: {self.package_name}"
+ )
+
+ def is_satisfied(self) -> bool:
+ return VersionComparison.from_string(self.version_comparison)(
+ version.parse(importlib.metadata.version(self.package_name)), version.parse(self.version)
+ )
+
+ def __repr__(self) -> str:
+ return f'Backend("{self.package_name}", {VersionComparison[self.version_comparison]}, "{self.version}")'
+
+ @property
+ def error_message(self):
+ return (
+ f"{{0}} requires the {self.package_name} library version {self.version_comparison}{self.version}. That"
+ f" library was not found with this version in your environment."
+ )
+
+
+def requires(*, backends=()):
+ """
+ This decorator enables two things:
+ - Attaching a `__backends` tuple to an object to see what are the necessary backends for it
+ to execute correctly without instantiating it
+ - The '@requires' string is used to dynamically import objects
+ """
+
+ if not isinstance(backends, tuple):
+ raise TypeError("Backends should be a tuple.")
+
+ applied_backends = []
+ for backend in backends:
+ if backend in BACKENDS_MAPPING:
+ applied_backends.append(backend)
+ else:
+ if any(key in backend for key in ["=", "<", ">"]):
+ applied_backends.append(Backend(backend))
+ else:
+ raise ValueError(f"Backend should be defined in the BACKENDS_MAPPING. Offending backend: {backend}")
+
+ def inner_fn(fun):
+ fun.__backends = applied_backends
+ return fun
+
+ return inner_fn
+
+
+BASE_FILE_REQUIREMENTS = {
+ lambda e: "modeling_tf_" in e: ("tf",),
+ lambda e: "modeling_flax_" in e: ("flax",),
+ lambda e: "modeling_" in e: ("torch",),
+ lambda e: e.startswith("tokenization_") and e.endswith("_fast"): ("tokenizers",),
+ lambda e: e.startswith("image_processing_") and e.endswith("_fast"): ("vision", "torch", "torchvision"),
+ lambda e: e.startswith("image_processing_"): ("vision",),
+}
+
+
+def fetch__all__(file_content) -> list[str]:
+ """
+ Returns the content of the __all__ variable in the file content.
+ Returns None if not defined, otherwise returns a list of strings.
+ """
+
+ if "__all__" not in file_content:
+ return []
+
+ start_index = None
+ lines = file_content.splitlines()
+ for index, line in enumerate(lines):
+ if line.startswith("__all__"):
+ start_index = index
+
+ # There is no line starting with `__all__`
+ if start_index is None:
+ return []
+
+ lines = lines[start_index:]
+
+ if not lines[0].startswith("__all__"):
+ raise ValueError(
+ "fetch__all__ accepts a list of lines, with the first line being the __all__ variable declaration"
+ )
+
+ # __all__ is defined on a single line
+ if lines[0].endswith("]"):
+ return [obj.strip("\"' ") for obj in lines[0].split("=")[1].strip(" []").split(",")]
+
+ # __all__ is defined on multiple lines
+ else:
+ _all: list[str] = []
+ for __all__line_index in range(1, len(lines)):
+ if lines[__all__line_index].strip() == "]":
+ return _all
+ else:
+ _all.append(lines[__all__line_index].strip("\"', "))
+
+ return _all
+
+
+@lru_cache
+def create_import_structure_from_path(module_path):
+ """
+ This method takes the path to a file/a folder and returns the import structure.
+ If a file is given, it will return the import structure of the parent folder.
+
+ Import structures are designed to be digestible by `_LazyModule` objects. They are
+ created from the __all__ definitions in each files as well as the `@require` decorators
+ above methods and objects.
+
+ The import structure allows explicit display of the required backends for a given object.
+ These backends are specified in two ways:
+
+ 1. Through their `@require`, if they are exported with that decorator. This `@require` decorator
+ accepts a `backend` tuple kwarg mentioning which backends are required to run this object.
+
+ 2. If an object is defined in a file with "default" backends, it will have, at a minimum, this
+ backend specified. The default backends are defined according to the filename:
+
+ - If a file is named like `modeling_*.py`, it will have a `torch` backend
+ - If a file is named like `modeling_tf_*.py`, it will have a `tf` backend
+ - If a file is named like `modeling_flax_*.py`, it will have a `flax` backend
+ - If a file is named like `tokenization_*_fast.py`, it will have a `tokenizers` backend
+ - If a file is named like `image_processing*_fast.py`, it will have a `torchvision` + `torch` backend
+
+ Backends serve the purpose of displaying a clear error message to the user in case the backends are not installed.
+ Should an object be imported without its required backends being in the environment, any attempt to use the
+ object will raise an error mentioning which backend(s) should be added to the environment in order to use
+ that object.
+
+ Here's an example of an input import structure at the src.transformers.models level:
+
+ {
+ 'albert': {
+ frozenset(): {
+ 'configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'}
+ },
+ frozenset({'tokenizers'}): {
+ 'tokenization_albert_fast': {'AlbertTokenizerFast'}
+ },
+ },
+ 'align': {
+ frozenset(): {
+ 'configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'},
+ 'processing_align': {'AlignProcessor'}
+ },
+ },
+ 'altclip': {
+ frozenset(): {
+ 'configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'},
+ 'processing_altclip': {'AltCLIPProcessor'},
+ }
+ }
+ }
+ """
+ import_structure = {}
+
+ if os.path.isfile(module_path):
+ module_path = os.path.dirname(module_path)
+
+ directory = module_path
+ adjacent_modules = []
+
+ for f in os.listdir(module_path):
+ if f != "__pycache__" and os.path.isdir(os.path.join(module_path, f)):
+ import_structure[f] = create_import_structure_from_path(os.path.join(module_path, f))
+
+ elif not os.path.isdir(os.path.join(directory, f)):
+ adjacent_modules.append(f)
+
+ # We're only taking a look at files different from __init__.py
+ # We could theoretically require things directly from the __init__.py
+ # files, but this is not supported at this time.
+ if "__init__.py" in adjacent_modules:
+ adjacent_modules.remove("__init__.py")
+
+ # Modular files should not be imported
+ def find_substring(substring, list_):
+ return any(substring in x for x in list_)
+
+ if find_substring("modular_", adjacent_modules) and find_substring("modeling_", adjacent_modules):
+ adjacent_modules = [module for module in adjacent_modules if "modular_" not in module]
+
+ module_requirements = {}
+ for module_name in adjacent_modules:
+ # Only modules ending in `.py` are accepted here.
+ if not module_name.endswith(".py"):
+ continue
+
+ with open(os.path.join(directory, module_name), encoding="utf-8") as f:
+ file_content = f.read()
+
+ # Remove the .py suffix
+ module_name = module_name[:-3]
+
+ previous_line = ""
+ previous_index = 0
+
+ # Some files have some requirements by default.
+ # For example, any file named `modeling_tf_xxx.py`
+ # should have TensorFlow as a required backend.
+ base_requirements = ()
+ for string_check, requirements in BASE_FILE_REQUIREMENTS.items():
+ if string_check(module_name):
+ base_requirements = requirements
+ break
+
+ # Objects that have a `@require` assigned to them will get exported
+ # with the backends specified in the decorator as well as the file backends.
+ exported_objects = set()
+ if "@requires" in file_content:
+ lines = file_content.split("\n")
+ for index, line in enumerate(lines):
+ # This allows exporting items with other decorators. We'll take a look
+ # at the line that follows at the same indentation level.
+ if line.startswith((" ", "\t", "@", ")")) and not line.startswith("@requires"):
+ continue
+
+ # Skipping line enables putting whatever we want between the
+ # export() call and the actual class/method definition.
+ # This is what enables having # Copied from statements, docs, etc.
+ skip_line = False
+
+ if "@requires" in previous_line:
+ skip_line = False
+
+ # Backends are defined on the same line as export
+ if "backends" in previous_line:
+ backends_string = previous_line.split("backends=")[1].split("(")[1].split(")")[0]
+ backends = tuple(sorted([b.strip("'\",") for b in backends_string.split(", ") if b]))
+
+ # Backends are defined in the lines following export, for example such as:
+ # @export(
+ # backends=(
+ # "sentencepiece",
+ # "torch",
+ # "tf",
+ # )
+ # )
+ #
+ # or
+ #
+ # @export(
+ # backends=(
+ # "sentencepiece", "tf"
+ # )
+ # )
+ elif "backends" in lines[previous_index + 1]:
+ backends = []
+ for backend_line in lines[previous_index:index]:
+ if "backends" in backend_line:
+ backend_line = backend_line.split("=")[1]
+ if '"' in backend_line or "'" in backend_line:
+ if ", " in backend_line:
+ backends.extend(backend.strip("()\"', ") for backend in backend_line.split(", "))
+ else:
+ backends.append(backend_line.strip("()\"', "))
+
+ # If the line is only a ')', then we reached the end of the backends and we break.
+ if backend_line.strip() == ")":
+ break
+ backends = tuple(backends)
+
+ # No backends are registered for export
+ else:
+ backends = ()
+
+ backends = frozenset(backends + base_requirements)
+ if backends not in module_requirements:
+ module_requirements[backends] = {}
+ if module_name not in module_requirements[backends]:
+ module_requirements[backends][module_name] = set()
+
+ if not line.startswith("class") and not line.startswith("def"):
+ skip_line = True
+ else:
+ start_index = 6 if line.startswith("class") else 4
+ object_name = line[start_index:].split("(")[0].strip(":")
+ module_requirements[backends][module_name].add(object_name)
+ exported_objects.add(object_name)
+
+ if not skip_line:
+ previous_line = line
+ previous_index = index
+
+ # All objects that are in __all__ should be exported by default.
+ # These objects are exported with the file backends.
+ if "__all__" in file_content:
+ for _all_object in fetch__all__(file_content):
+ if _all_object not in exported_objects:
+ backends = frozenset(base_requirements)
+ if backends not in module_requirements:
+ module_requirements[backends] = {}
+ if module_name not in module_requirements[backends]:
+ module_requirements[backends][module_name] = set()
+
+ module_requirements[backends][module_name].add(_all_object)
+
+ import_structure = {**module_requirements, **import_structure}
+ return import_structure
+
+
+def spread_import_structure(nested_import_structure):
+ """
+ This method takes as input an unordered import structure and brings the required backends at the top-level,
+ aggregating modules and objects under their required backends.
+
+ Here's an example of an input import structure at the src.transformers.models level:
+
+ {
+ 'albert': {
+ frozenset(): {
+ 'configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'}
+ },
+ frozenset({'tokenizers'}): {
+ 'tokenization_albert_fast': {'AlbertTokenizerFast'}
+ },
+ },
+ 'align': {
+ frozenset(): {
+ 'configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'},
+ 'processing_align': {'AlignProcessor'}
+ },
+ },
+ 'altclip': {
+ frozenset(): {
+ 'configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'},
+ 'processing_altclip': {'AltCLIPProcessor'},
+ }
+ }
+ }
+
+ Here's an example of an output import structure at the src.transformers.models level:
+
+ {
+ frozenset({'tokenizers'}): {
+ 'albert.tokenization_albert_fast': {'AlbertTokenizerFast'}
+ },
+ frozenset(): {
+ 'albert.configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'},
+ 'align.processing_align': {'AlignProcessor'},
+ 'align.configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'},
+ 'altclip.configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'},
+ 'altclip.processing_altclip': {'AltCLIPProcessor'}
+ }
+ }
+
+ """
+
+ def propagate_frozenset(unordered_import_structure):
+ frozenset_first_import_structure = {}
+ for _key, _value in unordered_import_structure.items():
+ # If the value is not a dict but a string, no need for custom manipulation
+ if not isinstance(_value, dict):
+ frozenset_first_import_structure[_key] = _value
+
+ elif any(isinstance(v, frozenset) for v in _value):
+ for k, v in _value.items():
+ if isinstance(k, frozenset):
+ # Here we want to switch around _key and k to propagate k upstream if it is a frozenset
+ if k not in frozenset_first_import_structure:
+ frozenset_first_import_structure[k] = {}
+ if _key not in frozenset_first_import_structure[k]:
+ frozenset_first_import_structure[k][_key] = {}
+
+ frozenset_first_import_structure[k][_key].update(v)
+
+ else:
+ # If k is not a frozenset, it means that the dictionary is not "level": some keys (top-level)
+ # are frozensets, whereas some are not -> frozenset keys are at an unknown depth-level of the
+ # dictionary.
+ #
+ # We recursively propagate the frozenset for this specific dictionary so that the frozensets
+ # are at the top-level when we handle them.
+ propagated_frozenset = propagate_frozenset({k: v})
+ for r_k, r_v in propagated_frozenset.items():
+ if isinstance(_key, frozenset):
+ if r_k not in frozenset_first_import_structure:
+ frozenset_first_import_structure[r_k] = {}
+ if _key not in frozenset_first_import_structure[r_k]:
+ frozenset_first_import_structure[r_k][_key] = {}
+
+ # _key is a frozenset -> we switch around the r_k and _key
+ frozenset_first_import_structure[r_k][_key].update(r_v)
+ else:
+ if _key not in frozenset_first_import_structure:
+ frozenset_first_import_structure[_key] = {}
+ if r_k not in frozenset_first_import_structure[_key]:
+ frozenset_first_import_structure[_key][r_k] = {}
+
+ # _key is not a frozenset -> we keep the order of r_k and _key
+ frozenset_first_import_structure[_key][r_k].update(r_v)
+
+ else:
+ frozenset_first_import_structure[_key] = propagate_frozenset(_value)
+
+ return frozenset_first_import_structure
+
+ def flatten_dict(_dict, previous_key=None):
+ items = []
+ for _key, _value in _dict.items():
+ _key = f"{previous_key}.{_key}" if previous_key is not None else _key
+ if isinstance(_value, dict):
+ items.extend(flatten_dict(_value, _key).items())
+ else:
+ items.append((_key, _value))
+ return dict(items)
+
+ # The tuples contain the necessary backends. We want these first, so we propagate them up the
+ # import structure.
+ ordered_import_structure = nested_import_structure
+
+ # 6 is a number that gives us sufficient depth to go through all files and foreseeable folder depths
+ # while not taking too long to parse.
+ for i in range(6):
+ ordered_import_structure = propagate_frozenset(ordered_import_structure)
+
+ # We then flatten the dict so that it references a module path.
+ flattened_import_structure = {}
+ for key, value in ordered_import_structure.copy().items():
+ if isinstance(key, str):
+ del ordered_import_structure[key]
+ else:
+ flattened_import_structure[key] = flatten_dict(value)
+
+ return flattened_import_structure
+
+
+@lru_cache
+def define_import_structure(module_path: str, prefix: Optional[str] = None) -> IMPORT_STRUCTURE_T:
+ """
+ This method takes a module_path as input and creates an import structure digestible by a _LazyModule.
+
+ Here's an example of an output import structure at the src.transformers.models level:
+
+ {
+ frozenset({'tokenizers'}): {
+ 'albert.tokenization_albert_fast': {'AlbertTokenizerFast'}
+ },
+ frozenset(): {
+ 'albert.configuration_albert': {'AlbertConfig', 'AlbertOnnxConfig'},
+ 'align.processing_align': {'AlignProcessor'},
+ 'align.configuration_align': {'AlignConfig', 'AlignTextConfig', 'AlignVisionConfig'},
+ 'altclip.configuration_altclip': {'AltCLIPConfig', 'AltCLIPTextConfig', 'AltCLIPVisionConfig'},
+ 'altclip.processing_altclip': {'AltCLIPProcessor'}
+ }
+ }
+
+ The import structure is a dict defined with frozensets as keys, and dicts of strings to sets of objects.
+
+ If `prefix` is not None, it will add that prefix to all keys in the returned dict.
+ """
+ import_structure = create_import_structure_from_path(module_path)
+ spread_dict = spread_import_structure(import_structure)
+
+ if prefix is None:
+ return spread_dict
+ else:
+ spread_dict = {k: {f"{prefix}.{kk}": vv for kk, vv in v.items()} for k, v in spread_dict.items()}
+ return spread_dict
+
+
+def clear_import_cache() -> None:
+ """
+ Clear cached Transformers modules to allow reloading modified code.
+
+ This is useful when actively developing/modifying Transformers code.
+ """
+ # Get all transformers modules
+ transformers_modules = [mod_name for mod_name in sys.modules if mod_name.startswith("transformers.")]
+
+ # Remove them from sys.modules
+ for mod_name in transformers_modules:
+ module = sys.modules[mod_name]
+ # Clear _LazyModule caches if applicable
+ if isinstance(module, _LazyModule):
+ module._objects = {} # Clear cached objects
+ del sys.modules[mod_name]
+
+ # Force reload main transformers module
+ if "transformers" in sys.modules:
+ main_module = sys.modules["transformers"]
+ if isinstance(main_module, _LazyModule):
+ main_module._objects = {} # Clear cached objects
+ importlib.reload(main_module)
diff --git a/phivenv/Lib/site-packages/transformers/utils/logging.py b/phivenv/Lib/site-packages/transformers/utils/logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..88a6a9769f65cc8d027059088cb02828e2fa13e8
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/logging.py
@@ -0,0 +1,409 @@
+# Copyright 2020 Optuna, Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Logging utilities."""
+
+import functools
+import logging
+import os
+import sys
+import threading
+from logging import (
+ CRITICAL, # NOQA
+ DEBUG, # NOQA
+ ERROR, # NOQA
+ FATAL, # NOQA
+ INFO, # NOQA
+ NOTSET, # NOQA
+ WARN, # NOQA
+ WARNING, # NOQA
+)
+from logging import captureWarnings as _captureWarnings
+from typing import Optional
+
+import huggingface_hub.utils as hf_hub_utils
+from tqdm import auto as tqdm_lib
+
+
+_lock = threading.Lock()
+_default_handler: Optional[logging.Handler] = None
+
+log_levels = {
+ "detail": logging.DEBUG, # will also print filename and line number
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL,
+}
+
+_default_log_level = logging.WARNING
+
+_tqdm_active = not hf_hub_utils.are_progress_bars_disabled()
+
+
+def _get_default_logging_level():
+ """
+ If TRANSFORMERS_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is
+ not - fall back to `_default_log_level`
+ """
+ env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)
+ if env_level_str:
+ if env_level_str in log_levels:
+ return log_levels[env_level_str]
+ else:
+ logging.getLogger().warning(
+ f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
+ f"has to be one of: {', '.join(log_levels.keys())}"
+ )
+ return _default_log_level
+
+
+def _get_library_name() -> str:
+ return __name__.split(".")[0]
+
+
+def _get_library_root_logger() -> logging.Logger:
+ return logging.getLogger(_get_library_name())
+
+
+def _configure_library_root_logger() -> None:
+ global _default_handler
+
+ with _lock:
+ if _default_handler:
+ # This library has already configured the library root logger.
+ return
+ _default_handler = logging.StreamHandler() # Set sys.stderr as stream.
+ # set defaults based on https://github.com/pyinstaller/pyinstaller/issues/7334#issuecomment-1357447176
+ if sys.stderr is None:
+ sys.stderr = open(os.devnull, "w")
+
+ _default_handler.flush = sys.stderr.flush
+
+ # Apply our default configuration to the library root logger.
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.addHandler(_default_handler)
+ library_root_logger.setLevel(_get_default_logging_level())
+ # if logging level is debug, we add pathname and lineno to formatter for easy debugging
+ if os.getenv("TRANSFORMERS_VERBOSITY", None) == "detail":
+ formatter = logging.Formatter("[%(levelname)s|%(pathname)s:%(lineno)s] %(asctime)s >> %(message)s")
+ _default_handler.setFormatter(formatter)
+
+ is_ci = os.getenv("CI") is not None and os.getenv("CI").upper() in {"1", "ON", "YES", "TRUE"}
+ library_root_logger.propagate = is_ci
+
+
+def _reset_library_root_logger() -> None:
+ global _default_handler
+
+ with _lock:
+ if not _default_handler:
+ return
+
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.removeHandler(_default_handler)
+ library_root_logger.setLevel(logging.NOTSET)
+ _default_handler = None
+
+
+def get_log_levels_dict():
+ return log_levels
+
+
+def captureWarnings(capture):
+ """
+ Calls the `captureWarnings` method from the logging library to enable management of the warnings emitted by the
+ `warnings` library.
+
+ Read more about this method here:
+ https://docs.python.org/3/library/logging.html#integration-with-the-warnings-module
+
+ All warnings will be logged through the `py.warnings` logger.
+
+ Careful: this method also adds a handler to this logger if it does not already have one, and updates the logging
+ level of that logger to the library's root logger.
+ """
+ logger = get_logger("py.warnings")
+
+ if not logger.handlers:
+ logger.addHandler(_default_handler)
+
+ logger.setLevel(_get_library_root_logger().level)
+
+ _captureWarnings(capture)
+
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+ """
+ Return a logger with the specified name.
+
+ This function is not supposed to be directly accessed unless you are writing a custom transformers module.
+ """
+
+ if name is None:
+ name = _get_library_name()
+
+ _configure_library_root_logger()
+ return logging.getLogger(name)
+
+
+def get_verbosity() -> int:
+ """
+ Return the current level for the 🤗 Transformers's root logger as an int.
+
+ Returns:
+ `int`: The logging level.
+
+
+
+ 🤗 Transformers has following logging levels:
+
+ - 50: `transformers.logging.CRITICAL` or `transformers.logging.FATAL`
+ - 40: `transformers.logging.ERROR`
+ - 30: `transformers.logging.WARNING` or `transformers.logging.WARN`
+ - 20: `transformers.logging.INFO`
+ - 10: `transformers.logging.DEBUG`
+
+ """
+
+ _configure_library_root_logger()
+ return _get_library_root_logger().getEffectiveLevel()
+
+
+def set_verbosity(verbosity: int) -> None:
+ """
+ Set the verbosity level for the 🤗 Transformers's root logger.
+
+ Args:
+ verbosity (`int`):
+ Logging level, e.g., one of:
+
+ - `transformers.logging.CRITICAL` or `transformers.logging.FATAL`
+ - `transformers.logging.ERROR`
+ - `transformers.logging.WARNING` or `transformers.logging.WARN`
+ - `transformers.logging.INFO`
+ - `transformers.logging.DEBUG`
+ """
+
+ _configure_library_root_logger()
+ _get_library_root_logger().setLevel(verbosity)
+
+
+def set_verbosity_info():
+ """Set the verbosity to the `INFO` level."""
+ return set_verbosity(INFO)
+
+
+def set_verbosity_warning():
+ """Set the verbosity to the `WARNING` level."""
+ return set_verbosity(WARNING)
+
+
+def set_verbosity_debug():
+ """Set the verbosity to the `DEBUG` level."""
+ return set_verbosity(DEBUG)
+
+
+def set_verbosity_error():
+ """Set the verbosity to the `ERROR` level."""
+ return set_verbosity(ERROR)
+
+
+def disable_default_handler() -> None:
+ """Disable the default handler of the HuggingFace Transformers's root logger."""
+
+ _configure_library_root_logger()
+
+ assert _default_handler is not None
+ _get_library_root_logger().removeHandler(_default_handler)
+
+
+def enable_default_handler() -> None:
+ """Enable the default handler of the HuggingFace Transformers's root logger."""
+
+ _configure_library_root_logger()
+
+ assert _default_handler is not None
+ _get_library_root_logger().addHandler(_default_handler)
+
+
+def add_handler(handler: logging.Handler) -> None:
+ """adds a handler to the HuggingFace Transformers's root logger."""
+
+ _configure_library_root_logger()
+
+ assert handler is not None
+ _get_library_root_logger().addHandler(handler)
+
+
+def remove_handler(handler: logging.Handler) -> None:
+ """removes given handler from the HuggingFace Transformers's root logger."""
+
+ _configure_library_root_logger()
+
+ assert handler is not None and handler not in _get_library_root_logger().handlers
+ _get_library_root_logger().removeHandler(handler)
+
+
+def disable_propagation() -> None:
+ """
+ Disable propagation of the library log outputs. Note that log propagation is disabled by default.
+ """
+
+ _configure_library_root_logger()
+ _get_library_root_logger().propagate = False
+
+
+def enable_propagation() -> None:
+ """
+ Enable propagation of the library log outputs. Please disable the HuggingFace Transformers's default handler to
+ prevent double logging if the root logger has been configured.
+ """
+
+ _configure_library_root_logger()
+ _get_library_root_logger().propagate = True
+
+
+def enable_explicit_format() -> None:
+ """
+ Enable explicit formatting for every HuggingFace Transformers's logger. The explicit formatter is as follows:
+ ```
+ [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE
+ ```
+ All handlers currently bound to the root logger are affected by this method.
+ """
+ handlers = _get_library_root_logger().handlers
+
+ for handler in handlers:
+ formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s")
+ handler.setFormatter(formatter)
+
+
+def reset_format() -> None:
+ """
+ Resets the formatting for HuggingFace Transformers's loggers.
+
+ All handlers currently bound to the root logger are affected by this method.
+ """
+ handlers = _get_library_root_logger().handlers
+
+ for handler in handlers:
+ handler.setFormatter(None)
+
+
+def warning_advice(self, *args, **kwargs):
+ """
+ This method is identical to `logger.warning()`, but if env var TRANSFORMERS_NO_ADVISORY_WARNINGS=1 is set, this
+ warning will not be printed
+ """
+ no_advisory_warnings = os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS")
+ if no_advisory_warnings:
+ return
+ self.warning(*args, **kwargs)
+
+
+logging.Logger.warning_advice = warning_advice
+
+
+@functools.lru_cache(None)
+def warning_once(self, *args, **kwargs):
+ """
+ This method is identical to `logger.warning()`, but will emit the warning with the same message only once
+
+ Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache.
+ The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to
+ another type of cache that includes the caller frame information in the hashing function.
+ """
+ self.warning(*args, **kwargs)
+
+
+logging.Logger.warning_once = warning_once
+
+
+@functools.lru_cache(None)
+def info_once(self, *args, **kwargs):
+ """
+ This method is identical to `logger.info()`, but will emit the info with the same message only once
+
+ Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache.
+ The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to
+ another type of cache that includes the caller frame information in the hashing function.
+ """
+ self.info(*args, **kwargs)
+
+
+logging.Logger.info_once = info_once
+
+
+class EmptyTqdm:
+ """Dummy tqdm which doesn't do anything."""
+
+ def __init__(self, *args, **kwargs): # pylint: disable=unused-argument
+ self._iterator = args[0] if args else None
+
+ def __iter__(self):
+ return iter(self._iterator)
+
+ def __getattr__(self, _):
+ """Return empty function."""
+
+ def empty_fn(*args, **kwargs): # pylint: disable=unused-argument
+ return
+
+ return empty_fn
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type_, value, traceback):
+ return
+
+
+class _tqdm_cls:
+ def __call__(self, *args, **kwargs):
+ if _tqdm_active:
+ return tqdm_lib.tqdm(*args, **kwargs)
+ else:
+ return EmptyTqdm(*args, **kwargs)
+
+ def set_lock(self, *args, **kwargs):
+ self._lock = None
+ if _tqdm_active:
+ return tqdm_lib.tqdm.set_lock(*args, **kwargs)
+
+ def get_lock(self):
+ if _tqdm_active:
+ return tqdm_lib.tqdm.get_lock()
+
+
+tqdm = _tqdm_cls()
+
+
+def is_progress_bar_enabled() -> bool:
+ """Return a boolean indicating whether tqdm progress bars are enabled."""
+ return bool(_tqdm_active)
+
+
+def enable_progress_bar():
+ """Enable tqdm progress bar."""
+ global _tqdm_active
+ _tqdm_active = True
+ hf_hub_utils.enable_progress_bars()
+
+
+def disable_progress_bar():
+ """Disable tqdm progress bar."""
+ global _tqdm_active
+ _tqdm_active = False
+ hf_hub_utils.disable_progress_bars()
diff --git a/phivenv/Lib/site-packages/transformers/utils/metrics.py b/phivenv/Lib/site-packages/transformers/utils/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..114abd8400e0e24f90a8a216bb2450a170887eb1
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/metrics.py
@@ -0,0 +1,416 @@
+import functools
+import logging
+import time
+from enum import Enum
+from typing import Any, Callable, Optional, Union
+
+import torch
+
+
+class RequestStatus(Enum):
+ """Status of a generation request through its lifecycle."""
+
+ PENDING = "pending"
+ PREFILLING = "prefilling"
+ PREFILLING_SPLIT = "prefilling_split"
+ SPLIT_PENDING_REMAINDER = "split_pending_remainder"
+ DECODING = "decoding"
+ FINISHED = "finished"
+ FAILED = "failed"
+
+
+try:
+ from opentelemetry import metrics
+ from opentelemetry.trace import Status, StatusCode, get_tracer
+
+ _has_opentelemetry = True
+except ImportError:
+ _has_opentelemetry = False
+
+
+def attach_tracer(tracer_name_template=None):
+ """
+ Decorator that attaches a tracer to a class.
+
+ This decorator should be applied to classes that need OpenTelemetry tracing.
+ It adds a tracer attribute to the class instance that can be used by the traced decorator.
+
+ Args:
+ tracer_name_template: Optional template string for the tracer name.
+ If provided, it should contain {module} which will be replaced with the class's full module path
+ and {class_name} for the class name.
+ If None, a default naming scheme will be used where:
+ - If the module already starts with "transformers.", it will use that directly
+ - Otherwise, it will prepend "transformers." to the module name
+
+ Returns:
+ Class decorator function
+ """
+ if not _has_opentelemetry:
+ return lambda cls: cls
+
+ def decorator(cls):
+ original_init = cls.__init__
+
+ @functools.wraps(original_init)
+ def init_with_tracer(self, *args, **kwargs):
+ original_init(self, *args, **kwargs)
+
+ module_name = cls.__module__
+ class_name = cls.__qualname__
+
+ if tracer_name_template is None:
+ if module_name.startswith("transformers."):
+ tracer_name = f"{module_name}.{class_name}"
+ else:
+ tracer_name = f"transformers.{module_name}.{class_name}"
+ else:
+ tracer_name = tracer_name_template.format(module=module_name, class_name=class_name)
+
+ self.tracer = get_tracer(tracer_name)
+
+ cls.__init__ = init_with_tracer
+ return cls
+
+ return decorator
+
+
+def traced(
+ func=None,
+ *,
+ span_name=None,
+ standalone=False,
+ additional_attributes: Optional[list[tuple[str, str, Union[Any, Callable[[Any], Any]]]]] = None,
+):
+ """
+ Decorator to trace function calls with OpenTelemetry.
+
+ Can be used as @traced or @traced(span_name="custom_name")
+
+ Args:
+ func: The function to trace
+ span_name: Optional custom name for the span (defaults to function name)
+ standalone: If True, creates a parentless span
+ additional_attributes: Optional list of additional attributes to set on the span.
+ Each item is a tuple of (instance_attribute_name, span_attribute_key, value_or_transform_function)
+ where:
+ - instance_attribute_name: Name of the attribute to get from the class instance
+ - span_attribute_key: Key to use when setting the attribute on the span
+ - value_or_transform_function: Either a raw value to use directly, or a function to transform
+ the attribute value before setting it on the span
+
+ Returns:
+ Decorated function with tracing
+ """
+
+ def decorator(func):
+ if not _has_opentelemetry:
+ return func
+
+ import functools
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ instance = args[0] if args and (hasattr(func, "__self__") and func.__self__ is not None) else None
+ is_method = instance is not None
+
+ if is_method and hasattr(instance, "tracer"):
+ tracer = instance.tracer
+ else:
+ tracer = get_tracer(f"transformers.{func.__module__}.{func.__name__}")
+
+ name = span_name or func.__name__
+ span_fn = tracer.start_span if standalone else tracer.start_as_current_span
+ with span_fn(name) as span:
+ span.set_attribute("function.name", func.__name__)
+ span.set_attribute("function.module", func.__module__)
+ span.set_attribute("function.is_method", is_method)
+
+ if args:
+ for i, arg in enumerate(args):
+ if isinstance(arg, (str, int, float, bool)) or arg is None:
+ span.set_attribute(f"args.{i}", str(arg))
+ else:
+ span.set_attribute(f"args.{i}", str(type(arg)))
+ if kwargs:
+ for key, value in kwargs.items():
+ if isinstance(value, (str, int, float, bool)) or value is None:
+ span.set_attribute(f"kwargs.{key}", str(value))
+ else:
+ span.set_attribute(f"kwargs.{key}", str(type(value)))
+
+ if additional_attributes and is_method:
+ for attr_config in additional_attributes:
+ instance_attribute_name, span_attribute_key, value_or_transform_function = attr_config
+ if hasattr(instance, instance_attribute_name):
+ attribute_value = getattr(instance, instance_attribute_name)
+ if callable(value_or_transform_function):
+ transformed_value = value_or_transform_function(attribute_value)
+ else:
+ transformed_value = value_or_transform_function
+ span.set_attribute(span_attribute_key, transformed_value)
+
+ try:
+ result = func(*args, **kwargs)
+ return result
+ except Exception as e:
+ span.set_status(Status(StatusCode.ERROR))
+ span.record_exception(e)
+ raise
+
+ return wrapper
+
+ if func is None:
+ return decorator
+ return decorator(func)
+
+
+logger = logging.getLogger(__name__)
+
+
+@attach_tracer()
+class ContinuousBatchProcessorMetrics:
+ """Metrics collection for ContinuousBatchProcessor."""
+
+ def __init__(self, max_batch_tokens: int):
+ """Initialize metrics for continuous batch processor.
+
+ Args:
+ max_batch_tokens: Maximum number of tokens in a batch
+ """
+ self.max_batch_tokens = max_batch_tokens
+
+ self._setup_metrics()
+
+ def _setup_metrics(self):
+ """Initialize OpenTelemetry metrics and tracing if the library is available."""
+
+ if not _has_opentelemetry:
+ logger.info("OpenTelemetry is not installed. Metrics and tracing will not be recorded.")
+ return
+
+ self.meter = metrics.get_meter("transformers.generation.continuous_batch_processor")
+
+ # Define appropriate buckets for TTFT (typically ranges from ~50ms to several seconds)
+ ttft_buckets = [10, 25, 50, 75, 100, 150, 200, 300, 500, 750, 1000, 2000, 5000, 10000]
+
+ self.ttft_histogram = self.meter.create_histogram(
+ name="ttft_milliseconds",
+ description="Time to first token in milliseconds",
+ unit="ms",
+ explicit_bucket_boundaries_advisory=ttft_buckets,
+ )
+
+ self.active_requests_gauge = self.meter.create_gauge(
+ name="active_requests_count",
+ description="Number of active requests currently being processed",
+ unit="requests",
+ )
+
+ self.waiting_requests_gauge = self.meter.create_gauge(
+ name="waiting_requests_count",
+ description="Number of requests waiting to be processed",
+ unit="requests",
+ )
+
+ # Define appropriate buckets for request latency (similar to TTFT but with higher upper bounds)
+ latency_buckets = [50, 100, 250, 500, 1000, 2000, 5000, 10000, 20000, 30000, 60000]
+
+ self.request_latency_histogram = self.meter.create_histogram(
+ name="request_latency_milliseconds",
+ description="End-to-end latency for completed requests in milliseconds",
+ unit="ms",
+ explicit_bucket_boundaries_advisory=latency_buckets,
+ )
+
+ self.decode_prefill_ratio_gauge = self.meter.create_gauge(
+ name="decode_prefill_ratio",
+ description="Ratio of decode tokens to prefill tokens in a batch",
+ unit="ratio",
+ )
+
+ self.prefill_tokens_counter = self.meter.create_counter(
+ name="prefill_tokens_processed",
+ description="Number of prefill tokens processed",
+ unit="tokens",
+ )
+
+ self.decode_tokens_counter = self.meter.create_counter(
+ name="decode_tokens_processed",
+ description="Number of decode tokens processed",
+ unit="tokens",
+ )
+
+ # Define appropriate buckets for batch fill percentage (0-100%)
+ batch_fill_buckets = [5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 98, 100]
+
+ self.batch_fill_percentage_histogram = self.meter.create_histogram(
+ name="batch_fill_percentage",
+ description="Percentage of max_batch_tokens utilized in each batch",
+ unit="percent",
+ explicit_bucket_boundaries_advisory=batch_fill_buckets,
+ )
+
+ self.kv_cache_free_memory_gauge = self.meter.create_gauge(
+ name="kv_cache_free_memory_bytes",
+ description="Free memory of the PagedAttentionCache in bytes",
+ unit="bytes",
+ )
+
+ self.kv_cache_memory_gauge = self.meter.create_gauge(
+ name="kv_cache_memory_bytes",
+ description="Memory usage of the PagedAttentionCache in bytes",
+ unit="bytes",
+ )
+
+ @traced
+ def record_ttft_metric(self, created_time: float, request_id: str) -> None:
+ """Record Time to First Token (TTFT).
+
+ Args:
+ created_time: The time the request was created
+ request_id: The ID of the request
+ """
+ if not _has_opentelemetry:
+ return
+
+ ttft_ms = (time.time() - created_time) * 1000.0
+
+ try:
+ self.ttft_histogram.record(ttft_ms)
+ logger.debug(f"Recorded TTFT for request {request_id}: {ttft_ms:.2f}ms")
+ except Exception as e:
+ logger.warning(f"Failed to record TTFT metric: {e}")
+
+ @traced
+ def record_batch_metrics(self, requests_in_batch: list) -> None:
+ """Record metrics about the batch composition including decode/prefill ratio and batch fill percentage.
+
+ Args:
+ requests_in_batch: List of request states in the current batch
+ """
+ if not _has_opentelemetry or not requests_in_batch:
+ return
+
+ decode_tokens = 0
+ prefill_tokens = 0
+
+ for state in requests_in_batch:
+ if state.status == RequestStatus.DECODING:
+ decode_tokens += 1
+ elif state.status in [RequestStatus.PREFILLING, RequestStatus.PREFILLING_SPLIT]:
+ prefill_tokens += len(state.prompt_ids)
+
+ total_batch_tokens = decode_tokens + prefill_tokens
+
+ try:
+ if prefill_tokens > 0:
+ self.prefill_tokens_counter.add(prefill_tokens)
+
+ if decode_tokens > 0:
+ self.decode_tokens_counter.add(decode_tokens)
+
+ if prefill_tokens > 0:
+ ratio = decode_tokens / prefill_tokens
+ self.decode_prefill_ratio_gauge.set(ratio)
+
+ fill_percentage = (total_batch_tokens / self.max_batch_tokens) * 100.0
+ self.batch_fill_percentage_histogram.record(fill_percentage)
+ logger.debug(
+ f"Batch metrics: {decode_tokens} decode tokens, {prefill_tokens} prefill tokens, "
+ f"batch fill: {fill_percentage:.2f}% ({total_batch_tokens}/{self.max_batch_tokens})"
+ )
+ except Exception as e:
+ logger.warning(f"Failed to record batch metrics: {e}")
+
+ @traced
+ def record_kv_cache_memory_metrics(self, cache) -> None:
+ """Record memory usage of the PagedAttentionCache without GPU synchronization.
+
+ This calculates the theoretical memory usage based on cache configuration
+ and the number of blocks currently in use.
+
+ Args:
+ cache: The PagedAttentionCache object to measure
+ """
+ if not _has_opentelemetry:
+ return
+
+ try:
+ # Calculate memory usage based on cache configuration
+ num_used_blocks = cache.num_blocks - len(cache._free_blocks)
+ num_layers = len(cache.key_cache)
+
+ # Each used block stores key and value states
+ # Each with shape: (num_kv_heads, block_size, head_dim)
+ bytes_per_parameter = 2 if cache.dtype in [torch.float16, torch.bfloat16] else 4 # Size in bytes
+
+ # Total bytes = num_layers * num_used_blocks * block_size *
+ # num_kv_heads * head_dim * 2 (both K and V) * bytes_per_parameter
+ memory_bytes = (
+ num_layers
+ * num_used_blocks
+ * cache.block_size
+ * cache.num_key_value_heads
+ * cache.head_dim
+ * 2 # For both key and value caches
+ * bytes_per_parameter
+ )
+
+ free_memory_bytes = (
+ num_layers
+ * len(cache._free_blocks)
+ * cache.block_size
+ * cache.num_key_value_heads
+ * cache.head_dim
+ * 2 # For both key and value caches
+ * bytes_per_parameter
+ )
+
+ self.kv_cache_memory_gauge.set(memory_bytes)
+ self.kv_cache_free_memory_gauge.set(free_memory_bytes)
+ logger.debug(
+ f"KV Cache memory: {memory_bytes / (1024 * 1024):.2f}MB, "
+ f"Used blocks: {num_used_blocks}/{cache.num_blocks} "
+ f"({num_used_blocks / cache.num_blocks * 100:.1f}%)"
+ )
+ except Exception as e:
+ logger.warning(f"Failed to record KV cache memory metrics: {e}")
+
+ @traced
+ def record_queue_metrics(self, active_requests: int, waiting_requests: int) -> None:
+ """Record metrics about active and waiting requests.
+
+ Args:
+ active_requests: Number of active requests
+ waiting_requests: Number of waiting requests
+ """
+ if not _has_opentelemetry:
+ return
+
+ try:
+ self.active_requests_gauge.set(active_requests)
+ self.waiting_requests_gauge.set(waiting_requests)
+ logger.debug(f"Queue metrics: {active_requests} active requests, {waiting_requests} waiting requests")
+ except Exception as e:
+ logger.warning(f"Failed to record queue metrics: {e}")
+
+ @traced
+ def record_request_completion(self, created_time: float, request_id: str) -> None:
+ """Record metrics about a completed request.
+
+ Args:
+ created_time: The time the request was created
+ request_id: The ID of the request
+ """
+ if not _has_opentelemetry:
+ return
+
+ latency_ms = (time.time() - created_time) * 1000.0
+
+ try:
+ self.request_latency_histogram.record(latency_ms)
+
+ logger.debug(f"Recorded request completion for {request_id}: {latency_ms:.2f}ms")
+ except Exception as e:
+ logger.warning(f"Failed to record request completion metric: {e}")
diff --git a/phivenv/Lib/site-packages/transformers/utils/model_parallel_utils.py b/phivenv/Lib/site-packages/transformers/utils/model_parallel_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7db16b70a75ccdce7bd9c07c5dfc99f42e5c8807
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/model_parallel_utils.py
@@ -0,0 +1,55 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from math import ceil
+
+
+def assert_device_map(device_map, num_blocks):
+ blocks = list(range(0, num_blocks))
+
+ device_map_blocks = [item for sublist in list(device_map.values()) for item in sublist]
+
+ # Duplicate check
+ duplicate_blocks = []
+ for i in device_map_blocks:
+ if device_map_blocks.count(i) > 1 and i not in duplicate_blocks:
+ duplicate_blocks.append(i)
+ # Missing blocks
+ missing_blocks = [i for i in blocks if i not in device_map_blocks]
+ extra_blocks = [i for i in device_map_blocks if i not in blocks]
+
+ if len(duplicate_blocks) != 0:
+ raise ValueError(
+ "Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device."
+ " These attention blocks were specified more than once: " + str(duplicate_blocks)
+ )
+ if len(missing_blocks) != 0:
+ raise ValueError(
+ "There are attention blocks for this model that are not specified in the device_map. Add these attention "
+ "blocks to a device on the device_map: " + str(missing_blocks)
+ )
+ if len(extra_blocks) != 0:
+ raise ValueError(
+ "The device_map contains more attention blocks than this model has. Remove these from the device_map:"
+ + str(extra_blocks)
+ )
+
+
+def get_device_map(n_layers, devices):
+ """Returns a dictionary of layers distributed evenly across all devices."""
+ layers = list(range(n_layers))
+ n_blocks = int(ceil(n_layers / len(devices)))
+ layers_list = [layers[i : i + n_blocks] for i in range(0, n_layers, n_blocks)]
+
+ return dict(zip(devices, layers_list))
diff --git a/phivenv/Lib/site-packages/transformers/utils/notebook.py b/phivenv/Lib/site-packages/transformers/utils/notebook.py
new file mode 100644
index 0000000000000000000000000000000000000000..397aa3e3ff044dbfdb869ff1166cf45f1861216f
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/notebook.py
@@ -0,0 +1,383 @@
+# Copyright 2020 Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import time
+from typing import Optional
+
+import IPython.display as disp
+
+from ..trainer_callback import TrainerCallback
+from ..trainer_utils import IntervalStrategy, has_length
+
+
+def format_time(t):
+ "Format `t` (in seconds) to (h):mm:ss"
+ t = int(t)
+ h, m, s = t // 3600, (t // 60) % 60, t % 60
+ return f"{h}:{m:02d}:{s:02d}" if h != 0 else f"{m:02d}:{s:02d}"
+
+
+def html_progress_bar(value, total, prefix, label, width=300):
+ # docstyle-ignore
+ return f"""
+
+ """
+
+
+def text_to_html_table(items):
+ "Put the texts in `items` in an HTML table."
+ html_code = """\n"""
+ html_code += """ \n \n"""
+ for i in items[0]:
+ html_code += f" | {i} | \n"
+ html_code += "
\n \n \n"
+ for line in items[1:]:
+ html_code += " \n"
+ for elt in line:
+ elt = f"{elt:.6f}" if isinstance(elt, float) else str(elt)
+ html_code += f" | {elt} | \n"
+ html_code += "
\n"
+ html_code += " \n
"
+ return html_code
+
+
+class NotebookProgressBar:
+ """
+ A progress par for display in a notebook.
+
+ Class attributes (overridden by derived classes)
+
+ - **warmup** (`int`) -- The number of iterations to do at the beginning while ignoring `update_every`.
+ - **update_every** (`float`) -- Since calling the time takes some time, we only do it every presumed
+ `update_every` seconds. The progress bar uses the average time passed up until now to guess the next value
+ for which it will call the update.
+
+ Args:
+ total (`int`):
+ The total number of iterations to reach.
+ prefix (`str`, *optional*):
+ A prefix to add before the progress bar.
+ leave (`bool`, *optional*, defaults to `True`):
+ Whether or not to leave the progress bar once it's completed. You can always call the
+ [`~utils.notebook.NotebookProgressBar.close`] method to make the bar disappear.
+ parent ([`~notebook.NotebookTrainingTracker`], *optional*):
+ A parent object (like [`~utils.notebook.NotebookTrainingTracker`]) that spawns progress bars and handle
+ their display. If set, the object passed must have a `display()` method.
+ width (`int`, *optional*, defaults to 300):
+ The width (in pixels) that the bar will take.
+
+ Example:
+
+ ```python
+ import time
+
+ pbar = NotebookProgressBar(100)
+ for val in range(100):
+ pbar.update(val)
+ time.sleep(0.07)
+ pbar.update(100)
+ ```"""
+
+ warmup = 5
+ update_every = 0.2
+
+ def __init__(
+ self,
+ total: int,
+ prefix: Optional[str] = None,
+ leave: bool = True,
+ parent: Optional["NotebookTrainingTracker"] = None,
+ width: int = 300,
+ ):
+ self.total = total
+ self.prefix = "" if prefix is None else prefix
+ self.leave = leave
+ self.parent = parent
+ self.width = width
+ self.last_value = None
+ self.comment = None
+ self.output = None
+ self.value = None
+ self.label = None
+ if "VSCODE_PID" in os.environ:
+ self.update_every = 0.5 # Adjusted for smooth updated as html rending is slow on VS Code
+ # This is the only adjustment required to optimize training html rending
+
+ def update(self, value: int, force_update: bool = False, comment: Optional[str] = None):
+ """
+ The main method to update the progress bar to `value`.
+
+ Args:
+ value (`int`):
+ The value to use. Must be between 0 and `total`.
+ force_update (`bool`, *optional*, defaults to `False`):
+ Whether or not to force and update of the internal state and display (by default, the bar will wait for
+ `value` to reach the value it predicted corresponds to a time of more than the `update_every` attribute
+ since the last update to avoid adding boilerplate).
+ comment (`str`, *optional*):
+ A comment to add on the left of the progress bar.
+ """
+ self.value = value
+ if comment is not None:
+ self.comment = comment
+ if self.last_value is None:
+ self.start_time = self.last_time = time.time()
+ self.start_value = self.last_value = value
+ self.elapsed_time = self.predicted_remaining = None
+ self.first_calls = self.warmup
+ self.wait_for = 1
+ self.update_bar(value)
+ elif value <= self.last_value and not force_update:
+ return
+ elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for, self.total):
+ if self.first_calls > 0:
+ self.first_calls -= 1
+ current_time = time.time()
+ self.elapsed_time = current_time - self.start_time
+ # We could have value = self.start_value if the update is called twixe with the same start value.
+ if value > self.start_value:
+ self.average_time_per_item = self.elapsed_time / (value - self.start_value)
+ else:
+ self.average_time_per_item = None
+ if value >= self.total:
+ value = self.total
+ self.predicted_remaining = None
+ if not self.leave:
+ self.close()
+ elif self.average_time_per_item is not None:
+ self.predicted_remaining = self.average_time_per_item * (self.total - value)
+ self.update_bar(value)
+ self.last_value = value
+ self.last_time = current_time
+ if (self.average_time_per_item is None) or (self.average_time_per_item == 0):
+ self.wait_for = 1
+ else:
+ self.wait_for = max(int(self.update_every / self.average_time_per_item), 1)
+
+ def update_bar(self, value, comment=None):
+ spaced_value = " " * (len(str(self.total)) - len(str(value))) + str(value)
+ if self.elapsed_time is None:
+ self.label = f"[{spaced_value}/{self.total} : < :"
+ elif self.predicted_remaining is None:
+ self.label = f"[{spaced_value}/{self.total} {format_time(self.elapsed_time)}"
+ else:
+ self.label = (
+ f"[{spaced_value}/{self.total} {format_time(self.elapsed_time)} <"
+ f" {format_time(self.predicted_remaining)}"
+ )
+ if self.average_time_per_item == 0:
+ self.label += ", +inf it/s"
+ else:
+ self.label += f", {1 / self.average_time_per_item:.2f} it/s"
+
+ self.label += "]" if self.comment is None or len(self.comment) == 0 else f", {self.comment}]"
+ self.display()
+
+ def display(self):
+ self.html_code = html_progress_bar(self.value, self.total, self.prefix, self.label, self.width)
+ if self.parent is not None:
+ # If this is a child bar, the parent will take care of the display.
+ self.parent.display()
+ return
+ if self.output is None:
+ self.output = disp.display(disp.HTML(self.html_code), display_id=True)
+ else:
+ self.output.update(disp.HTML(self.html_code))
+
+ def close(self):
+ "Closes the progress bar."
+ if self.parent is None and self.output is not None:
+ self.output.update(disp.HTML(""))
+
+
+class NotebookTrainingTracker(NotebookProgressBar):
+ """
+ An object tracking the updates of an ongoing training with progress bars and a nice table reporting metrics.
+
+ Args:
+ num_steps (`int`): The number of steps during training. column_names (`list[str]`, *optional*):
+ The list of column names for the metrics table (will be inferred from the first call to
+ [`~utils.notebook.NotebookTrainingTracker.write_line`] if not set).
+ """
+
+ def __init__(self, num_steps, column_names=None):
+ super().__init__(num_steps)
+ self.inner_table = None if column_names is None else [column_names]
+ self.child_bar = None
+
+ def display(self):
+ self.html_code = html_progress_bar(self.value, self.total, self.prefix, self.label, self.width)
+ if self.inner_table is not None:
+ self.html_code += text_to_html_table(self.inner_table)
+ if self.child_bar is not None:
+ self.html_code += self.child_bar.html_code
+ if self.output is None:
+ self.output = disp.display(disp.HTML(self.html_code), display_id=True)
+ else:
+ self.output.update(disp.HTML(self.html_code))
+
+ def write_line(self, values):
+ """
+ Write the values in the inner table.
+
+ Args:
+ values (`dict[str, float]`): The values to display.
+ """
+ if self.inner_table is None:
+ self.inner_table = [list(values.keys()), list(values.values())]
+ else:
+ columns = self.inner_table[0]
+ for key in values:
+ if key not in columns:
+ columns.append(key)
+ self.inner_table[0] = columns
+ if len(self.inner_table) > 1:
+ last_values = self.inner_table[-1]
+ first_column = self.inner_table[0][0]
+ if last_values[0] != values[first_column]:
+ # write new line
+ self.inner_table.append([values.get(c, "No Log") for c in columns])
+ else:
+ # update last line
+ new_values = values
+ for c in columns:
+ if c not in new_values:
+ new_values[c] = last_values[columns.index(c)]
+ self.inner_table[-1] = [new_values[c] for c in columns]
+ else:
+ self.inner_table.append([values[c] for c in columns])
+
+ def add_child(self, total, prefix=None, width=300):
+ """
+ Add a child progress bar displayed under the table of metrics. The child progress bar is returned (so it can be
+ easily updated).
+
+ Args:
+ total (`int`): The number of iterations for the child progress bar.
+ prefix (`str`, *optional*): A prefix to write on the left of the progress bar.
+ width (`int`, *optional*, defaults to 300): The width (in pixels) of the progress bar.
+ """
+ self.child_bar = NotebookProgressBar(total, prefix=prefix, parent=self, width=width)
+ return self.child_bar
+
+ def remove_child(self):
+ """
+ Closes the child progress bar.
+ """
+ self.child_bar = None
+ self.display()
+
+
+class NotebookProgressCallback(TrainerCallback):
+ """
+ A [`TrainerCallback`] that displays the progress of training or evaluation, optimized for Jupyter Notebooks or
+ Google colab.
+ """
+
+ def __init__(self):
+ self.training_tracker = None
+ self.prediction_bar = None
+ self._force_next_update = False
+
+ def on_train_begin(self, args, state, control, **kwargs):
+ self.first_column = "Epoch" if args.eval_strategy == IntervalStrategy.EPOCH else "Step"
+ self.training_loss = 0
+ self.last_log = 0
+ column_names = [self.first_column] + ["Training Loss"]
+ if args.eval_strategy != IntervalStrategy.NO:
+ column_names.append("Validation Loss")
+ self.training_tracker = NotebookTrainingTracker(state.max_steps, column_names)
+
+ def on_step_end(self, args, state, control, **kwargs):
+ epoch = int(state.epoch) if int(state.epoch) == state.epoch else f"{state.epoch:.2f}"
+ self.training_tracker.update(
+ state.global_step + 1,
+ comment=f"Epoch {epoch}/{state.num_train_epochs}",
+ force_update=self._force_next_update,
+ )
+ self._force_next_update = False
+
+ def on_prediction_step(self, args, state, control, eval_dataloader=None, **kwargs):
+ if not has_length(eval_dataloader):
+ return
+ if self.prediction_bar is None:
+ if self.training_tracker is not None:
+ self.prediction_bar = self.training_tracker.add_child(len(eval_dataloader))
+ else:
+ self.prediction_bar = NotebookProgressBar(len(eval_dataloader))
+ self.prediction_bar.update(1)
+ else:
+ self.prediction_bar.update(self.prediction_bar.value + 1)
+
+ def on_predict(self, args, state, control, **kwargs):
+ if self.prediction_bar is not None:
+ self.prediction_bar.close()
+ self.prediction_bar = None
+
+ def on_log(self, args, state, control, logs=None, **kwargs):
+ # Only for when there is no evaluation
+ if args.eval_strategy == IntervalStrategy.NO and "loss" in logs:
+ values = {"Training Loss": logs["loss"]}
+ # First column is necessarily Step sine we're not in epoch eval strategy
+ values["Step"] = state.global_step
+ self.training_tracker.write_line(values)
+
+ def on_evaluate(self, args, state, control, metrics=None, **kwargs):
+ if self.training_tracker is not None:
+ values = {"Training Loss": "No log", "Validation Loss": "No log"}
+ for log in reversed(state.log_history):
+ if "loss" in log:
+ values["Training Loss"] = log["loss"]
+ break
+
+ if self.first_column == "Epoch":
+ values["Epoch"] = int(state.epoch)
+ else:
+ values["Step"] = state.global_step
+ metric_key_prefix = "eval"
+ for k in metrics:
+ if k.endswith("_loss"):
+ metric_key_prefix = re.sub(r"\_loss$", "", k)
+ _ = metrics.pop("total_flos", None)
+ _ = metrics.pop("epoch", None)
+ _ = metrics.pop(f"{metric_key_prefix}_runtime", None)
+ _ = metrics.pop(f"{metric_key_prefix}_samples_per_second", None)
+ _ = metrics.pop(f"{metric_key_prefix}_steps_per_second", None)
+ _ = metrics.pop(f"{metric_key_prefix}_jit_compilation_time", None)
+ for k, v in metrics.items():
+ splits = k.split("_")
+ name = " ".join([part.capitalize() for part in splits[1:]])
+ if name == "Loss":
+ # Single dataset
+ name = "Validation Loss"
+ values[name] = v
+ self.training_tracker.write_line(values)
+ self.training_tracker.remove_child()
+ self.prediction_bar = None
+ # Evaluation takes a long time so we should force the next update.
+ self._force_next_update = True
+
+ def on_train_end(self, args, state, control, **kwargs):
+ self.training_tracker.update(
+ state.global_step,
+ comment=f"Epoch {int(state.epoch)}/{state.num_train_epochs}",
+ force_update=True,
+ )
+ self.training_tracker = None
diff --git a/phivenv/Lib/site-packages/transformers/utils/peft_utils.py b/phivenv/Lib/site-packages/transformers/utils/peft_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3976acf168bee27583aa18510c1f0b28aeba00e
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/peft_utils.py
@@ -0,0 +1,125 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import importlib
+import os
+from typing import Optional, Union
+
+from packaging import version
+
+from .hub import cached_file
+from .import_utils import is_peft_available
+
+
+ADAPTER_CONFIG_NAME = "adapter_config.json"
+ADAPTER_WEIGHTS_NAME = "adapter_model.bin"
+ADAPTER_SAFE_WEIGHTS_NAME = "adapter_model.safetensors"
+
+
+def find_adapter_config_file(
+ model_id: str,
+ cache_dir: Optional[Union[str, os.PathLike]] = None,
+ force_download: bool = False,
+ resume_download: Optional[bool] = None,
+ proxies: Optional[dict[str, str]] = None,
+ token: Optional[Union[bool, str]] = None,
+ revision: Optional[str] = None,
+ local_files_only: bool = False,
+ subfolder: str = "",
+ _commit_hash: Optional[str] = None,
+) -> Optional[str]:
+ r"""
+ Simply checks if the model stored on the Hub or locally is an adapter model or not, return the path of the adapter
+ config file if it is, None otherwise.
+
+ Args:
+ model_id (`str`):
+ The identifier of the model to look for, can be either a local path or an id to the repository on the Hub.
+ cache_dir (`str` or `os.PathLike`, *optional*):
+ Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
+ cache should not be used.
+ force_download (`bool`, *optional*, defaults to `False`):
+ Whether or not to force to (re-)download the configuration files and override the cached versions if they
+ exist.
+ resume_download:
+ Deprecated and ignored. All downloads are now resumed by default when possible.
+ Will be removed in v5 of Transformers.
+ proxies (`dict[str, str]`, *optional*):
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ token (`str` or *bool*, *optional*):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `hf auth login` (stored in `~/.huggingface`).
+ revision (`str`, *optional*, defaults to `"main"`):
+ The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
+ git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
+ identifier allowed by git.
+
+
+
+ To test a pull request you made on the Hub, you can pass `revision="refs/pr/".
+
+
+
+ local_files_only (`bool`, *optional*, defaults to `False`):
+ If `True`, will only try to load the tokenizer configuration from local files.
+ subfolder (`str`, *optional*, defaults to `""`):
+ In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can
+ specify the folder name here.
+ """
+ adapter_cached_filename = None
+ if model_id is None:
+ return None
+ elif os.path.isdir(model_id):
+ list_remote_files = os.listdir(model_id)
+ if ADAPTER_CONFIG_NAME in list_remote_files:
+ adapter_cached_filename = os.path.join(model_id, ADAPTER_CONFIG_NAME)
+ else:
+ adapter_cached_filename = cached_file(
+ model_id,
+ ADAPTER_CONFIG_NAME,
+ cache_dir=cache_dir,
+ force_download=force_download,
+ resume_download=resume_download,
+ proxies=proxies,
+ token=token,
+ revision=revision,
+ local_files_only=local_files_only,
+ subfolder=subfolder,
+ _commit_hash=_commit_hash,
+ _raise_exceptions_for_gated_repo=False,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+
+ return adapter_cached_filename
+
+
+def check_peft_version(min_version: str) -> None:
+ r"""
+ Checks if the version of PEFT is compatible.
+
+ Args:
+ version (`str`):
+ The version of PEFT to check against.
+ """
+ if not is_peft_available():
+ raise ValueError("PEFT is not installed. Please install it with `pip install peft`")
+
+ is_peft_version_compatible = version.parse(importlib.metadata.version("peft")) >= version.parse(min_version)
+
+ if not is_peft_version_compatible:
+ raise ValueError(
+ f"The version of PEFT you are using is not compatible, please use a version that is greater"
+ f" than {min_version}"
+ )
diff --git a/phivenv/Lib/site-packages/transformers/utils/quantization_config.py b/phivenv/Lib/site-packages/transformers/utils/quantization_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..037bf3ed73d48c9e55a2d7ef364927822ed899ef
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/quantization_config.py
@@ -0,0 +1,2086 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+# Modifications Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+import dataclasses
+import importlib.metadata
+import json
+import os
+from dataclasses import dataclass, is_dataclass
+from enum import Enum
+from inspect import Parameter, signature
+from typing import Any, Optional, Union
+
+from packaging import version
+
+from ..utils import (
+ is_auto_awq_available,
+ is_compressed_tensors_available,
+ is_gptqmodel_available,
+ is_hqq_available,
+ is_quark_available,
+ is_torch_available,
+ is_torchao_available,
+ logging,
+)
+from .import_utils import is_auto_gptq_available
+
+
+if is_torch_available():
+ import torch
+
+logger = logging.get_logger(__name__)
+
+
+class QuantizationMethod(str, Enum):
+ BITS_AND_BYTES = "bitsandbytes"
+ GPTQ = "gptq"
+ AWQ = "awq"
+ AQLM = "aqlm"
+ VPTQ = "vptq"
+ QUANTO = "quanto"
+ EETQ = "eetq"
+ HIGGS = "higgs"
+ HQQ = "hqq"
+ COMPRESSED_TENSORS = "compressed-tensors"
+ FBGEMM_FP8 = "fbgemm_fp8"
+ TORCHAO = "torchao"
+ BITNET = "bitnet"
+ SPQR = "spqr"
+ FP8 = "fp8"
+ QUARK = "quark"
+ FPQUANT = "fp_quant"
+ AUTOROUND = "auto-round"
+ MXFP4 = "mxfp4"
+
+
+class AWQLinearVersion(str, Enum):
+ GEMM = "gemm"
+ GEMV = "gemv"
+ EXLLAMA = "exllama"
+ IPEX = "ipex"
+
+ @staticmethod
+ def from_str(version: str):
+ version = version.lower()
+ if version == "gemm":
+ return AWQLinearVersion.GEMM
+ elif version == "gemv":
+ return AWQLinearVersion.GEMV
+ elif version == "exllama":
+ return AWQLinearVersion.EXLLAMA
+ elif version == "ipex":
+ return AWQLinearVersion.IPEX
+ else:
+ raise ValueError(f"Unknown AWQLinearVersion {version}")
+
+
+class AwqBackendPackingMethod(str, Enum):
+ AUTOAWQ = "autoawq"
+ LLMAWQ = "llm-awq"
+
+
+@dataclass
+class QuantizationConfigMixin:
+ """
+ Mixin class for quantization config
+ """
+
+ quant_method: QuantizationMethod
+
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ """
+ Instantiates a [`QuantizationConfigMixin`] from a Python dictionary of parameters.
+
+ Args:
+ config_dict (`dict[str, Any]`):
+ Dictionary that will be used to instantiate the configuration object.
+ return_unused_kwargs (`bool`,*optional*, defaults to `False`):
+ Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in
+ `PreTrainedModel`.
+ kwargs (`dict[str, Any]`):
+ Additional parameters from which to initialize the configuration object.
+
+ Returns:
+ [`QuantizationConfigMixin`]: The configuration object instantiated from those parameters.
+ """
+ config = cls(**config_dict)
+
+ to_remove = []
+ for key, value in kwargs.items():
+ if hasattr(config, key):
+ setattr(config, key, value)
+ to_remove.append(key)
+ for key in to_remove:
+ kwargs.pop(key, None)
+
+ if return_unused_kwargs:
+ return config, kwargs
+ else:
+ return config
+
+ def to_json_file(self, json_file_path: Union[str, os.PathLike]):
+ """
+ Save this instance to a JSON file.
+
+ Args:
+ json_file_path (`str` or `os.PathLike`):
+ Path to the JSON file in which this configuration instance's parameters will be saved.
+ use_diff (`bool`, *optional*, defaults to `True`):
+ If set to `True`, only the difference between the config instance and the default
+ `QuantizationConfig()` is serialized to JSON file.
+ """
+ with open(json_file_path, "w", encoding="utf-8") as writer:
+ config_dict = self.to_dict()
+ json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
+
+ writer.write(json_string)
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary. Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+ return copy.deepcopy(self.__dict__)
+
+ def __iter__(self):
+ """allows `dict(obj)` for situations where obj may be a dict or QuantizationConfigMixin"""
+ for attr, value in copy.deepcopy(self.__dict__).items():
+ yield attr, value
+
+ def __repr__(self):
+ return f"{self.__class__.__name__} {self.to_json_string()}"
+
+ def to_json_string(self, use_diff: bool = True) -> str:
+ """
+ Serializes this instance to a JSON string.
+
+ Args:
+ use_diff (`bool`, *optional*, defaults to `True`):
+ If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`
+ is serialized to JSON string.
+
+ Returns:
+ `str`: String containing all the attributes that make up this configuration instance in JSON format.
+ """
+ if use_diff is True:
+ config_dict = self.to_diff_dict()
+ else:
+ config_dict = self.to_dict()
+ return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
+
+ def update(self, **kwargs):
+ """
+ Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes,
+ returning all the unused kwargs.
+
+ Args:
+ kwargs (`dict[str, Any]`):
+ Dictionary of attributes to tentatively update this class.
+
+ Returns:
+ `dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance.
+ """
+ to_remove = []
+ for key, value in kwargs.items():
+ if hasattr(self, key):
+ setattr(self, key, value)
+ to_remove.append(key)
+
+ # Remove all the attributes that were updated, without modifying the input dict
+ unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove}
+ return unused_kwargs
+
+
+@dataclass
+class AutoRoundConfig(QuantizationConfigMixin):
+ """This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded AutoRound quantization.
+
+ Args:
+ bits (`int`, *optional*, defaults to 4):
+ The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
+ group_size (`int`, *optional*, defaults to 128): Group-size value
+ sym (`bool`, *optional*, defaults to `True`): Symmetric quantization or not
+ backend (`str`, *optional*, defaults to `"auto"`): The kernel to use, e.g., ipex,marlin, exllamav2, triton, etc. Ref. https://github.com/intel/auto-round?tab=readme-ov-file#specify-backend
+ """
+
+ def __init__(
+ self,
+ bits: int = 4,
+ group_size: int = 128,
+ sym: bool = True,
+ backend: str = "auto",
+ **kwargs,
+ ):
+ self.bits = bits
+ self.group_size = group_size
+ self.sym = sym
+ self.backend = backend
+ self.packing_format = "auto_round:gptq"
+ if kwargs is not None:
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+ self.quant_method = QuantizationMethod.AUTOROUND
+ self.post_init()
+
+ def post_init(self):
+ r"""Safety checker that arguments are correct."""
+ if self.bits not in [2, 3, 4, 8]:
+ raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")
+ if self.group_size != -1 and self.group_size <= 0:
+ raise ValueError("group_size must be greater than 0 or equal to -1")
+
+ def get_loading_attributes(self):
+ loading_attributes_dict = {"backend": self.backend}
+ return loading_attributes_dict
+
+ def to_dict(self):
+ config_dict = super().to_dict()
+ return config_dict
+
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ quant_method = config_dict["quant_method"]
+ if "auto-round" not in quant_method and "gptq" not in quant_method and "awq" not in quant_method:
+ raise NotImplementedError(
+ "Failed to convert to auto_round format. Only `gptqv1`, `awq`, and `auto-round` formats are supported."
+ )
+
+ if "gptq" in quant_method and "meta" in config_dict:
+ raise NotImplementedError("Failed to convert gptq format to auto_round format. Only supports `gptqv1`")
+
+ if "awq" in quant_method and config_dict.get("version", "gemm") != "gemm":
+ raise NotImplementedError(
+ "Failed to convert awq format to auto_round format. Only supports awq format with gemm version"
+ )
+
+ if "auto-round" not in quant_method:
+ config_dict["packing_format"] = f"auto_round:{quant_method}"
+
+ return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)
+
+
+@dataclass
+class HqqConfig(QuantizationConfigMixin):
+ """
+ This is wrapper around hqq's BaseQuantizeConfig.
+
+ Args:
+ nbits (`int`, *optional*, defaults to 4):
+ Number of bits. Supported values are (8, 4, 3, 2, 1).
+ group_size (`int`, *optional*, defaults to 64):
+ Group-size value. Supported values are any value that is divisible by weight.shape[axis]).
+ view_as_float (`bool`, *optional*, defaults to `False`):
+ View the quantized weight as float (used in distributed training) if set to `True`.
+ axis (`Optional[int]`, *optional*):
+ Axis along which grouping is performed. Supported values are 0 or 1.
+ dynamic_config (dict, *optional*):
+ Parameters for dynamic configuration. The key is the name tag of the layer and the value is a quantization config.
+ If set, each layer specified by its id will use its dedicated quantization configuration.
+ skip_modules (`list[str]`, *optional*, defaults to `['lm_head']`):
+ List of `nn.Linear` layers to skip.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional parameters from which to initialize the configuration object.
+ """
+
+ def __init__(
+ self,
+ nbits: int = 4,
+ group_size: int = 64,
+ view_as_float: bool = False,
+ axis: Optional[int] = None,
+ dynamic_config: Optional[dict] = None,
+ skip_modules: list[str] = ["lm_head"],
+ **kwargs,
+ ):
+ if is_hqq_available():
+ from hqq.core.quantize import BaseQuantizeConfig as HQQBaseQuantizeConfig
+ else:
+ raise ImportError(
+ "A valid HQQ version (>=0.2.1) is not available. Please follow the instructions to install it: `https://github.com/mobiusml/hqq/`."
+ )
+
+ for deprecated_key in ["quant_zero", "quant_scale", "offload_meta"]:
+ if deprecated_key in kwargs:
+ logger.info(
+ deprecated_key + " is deprecated. This parameter will be ignored in quantization settings."
+ )
+
+ if axis is None:
+ axis = 1
+ logger.info("Setting axis=1 as faster backends such as TorchAO or BitBlas are only compatible with it.")
+
+ if axis not in [0, 1]:
+ raise ValueError("Invalid axis value. Only 0 and 1 are allowed.")
+
+ if dynamic_config is not None:
+ self.quant_config = {}
+ for key in dynamic_config:
+ self.quant_config[key] = HQQBaseQuantizeConfig(**dynamic_config[key])
+ else:
+ self.quant_config = HQQBaseQuantizeConfig(
+ **{
+ "nbits": nbits,
+ "group_size": group_size,
+ "view_as_float": view_as_float,
+ "axis": axis,
+ }
+ )
+
+ self.quant_method = QuantizationMethod.HQQ
+ self.skip_modules = skip_modules
+
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ pass
+
+ @classmethod
+ def from_dict(cls, config: dict[str, Any]):
+ """
+ Override from_dict, used in AutoQuantizationConfig.from_dict in quantizers/auto.py
+ """
+ instance = cls()
+ instance.quant_config = config["quant_config"]
+ instance.skip_modules = config["skip_modules"]
+ return instance
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary. Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+ return {
+ "quant_config": self.quant_config,
+ "quant_method": self.quant_method,
+ "skip_modules": self.skip_modules,
+ }
+
+ def __repr__(self):
+ config_dict = self.to_dict()
+ return f"{self.__class__.__name__} {json.dumps(config_dict, indent=2, sort_keys=True)}\n"
+
+ def to_diff_dict(self) -> dict[str, Any]:
+ """
+ Removes all attributes from config which correspond to the default config attributes for better readability and
+ serializes to a Python dictionary.
+ Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ config_dict = self.to_dict()
+
+ # get the default config dict
+ default_config_dict = HqqConfig().to_dict()
+
+ serializable_config_dict = {}
+
+ # only serialize values that differ from the default config
+ for key, value in config_dict.items():
+ if value != default_config_dict[key]:
+ serializable_config_dict[key] = value
+
+ return serializable_config_dict
+
+
+@dataclass
+class BitsAndBytesConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using `bitsandbytes`.
+
+ This replaces `load_in_8bit` or `load_in_4bit`therefore both options are mutually exclusive.
+
+ Currently only supports `LLM.int8()`, `FP4`, and `NF4` quantization. If more methods are added to `bitsandbytes`,
+ then more arguments will be added to this class.
+
+ Args:
+ load_in_8bit (`bool`, *optional*, defaults to `False`):
+ This flag is used to enable 8-bit quantization with LLM.int8().
+ load_in_4bit (`bool`, *optional*, defaults to `False`):
+ This flag is used to enable 4-bit quantization by replacing the Linear layers with FP4/NF4 layers from
+ `bitsandbytes`.
+ llm_int8_threshold (`float`, *optional*, defaults to 6.0):
+ This corresponds to the outlier threshold for outlier detection as described in `LLM.int8() : 8-bit Matrix
+ Multiplication for Transformers at Scale` paper: https://huggingface.co/papers/2208.07339 Any hidden states value
+ that is above this threshold will be considered an outlier and the operation on those values will be done
+ in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but
+ there are some exceptional systematic outliers that are very differently distributed for large models.
+ These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of
+ magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6,
+ but a lower threshold might be needed for more unstable models (small models, fine-tuning).
+ llm_int8_skip_modules (`list[str]`, *optional*):
+ An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as
+ Jukebox that has several heads in different places and not necessarily at the last position. For example
+ for `CausalLM` models, the last `lm_head` is kept in its original `dtype`.
+ llm_int8_enable_fp32_cpu_offload (`bool`, *optional*, defaults to `False`):
+ This flag is used for advanced use cases and users that are aware of this feature. If you want to split
+ your model in different parts and run some parts in int8 on GPU and some parts in fp32 on CPU, you can use
+ this flag. This is useful for offloading large models such as `google/flan-t5-xxl`. Note that the int8
+ operations will not be run on CPU.
+ llm_int8_has_fp16_weight (`bool`, *optional*, defaults to `False`):
+ This flag runs LLM.int8() with 16-bit main weights. This is useful for fine-tuning as the weights do not
+ have to be converted back and forth for the backward pass.
+ bnb_4bit_compute_dtype (`torch.dtype` or str, *optional*, defaults to `torch.float32`):
+ This sets the computational type which might be different than the input type. For example, inputs might be
+ fp32, but computation can be set to bf16 for speedups.
+ bnb_4bit_quant_type (`str`, *optional*, defaults to `"fp4"`):
+ This sets the quantization data type in the bnb.nn.Linear4Bit layers. Options are FP4 and NF4 data types
+ which are specified by `fp4` or `nf4`.
+ bnb_4bit_use_double_quant (`bool`, *optional*, defaults to `False`):
+ This flag is used for nested quantization where the quantization constants from the first quantization are
+ quantized again.
+ bnb_4bit_quant_storage (`torch.dtype` or str, *optional*, defaults to `torch.uint8`):
+ This sets the storage type to pack the quantized 4-bit params.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional parameters from which to initialize the configuration object.
+ """
+
+ def __init__(
+ self,
+ load_in_8bit=False,
+ load_in_4bit=False,
+ llm_int8_threshold=6.0,
+ llm_int8_skip_modules=None,
+ llm_int8_enable_fp32_cpu_offload=False,
+ llm_int8_has_fp16_weight=False,
+ bnb_4bit_compute_dtype=None,
+ bnb_4bit_quant_type="fp4",
+ bnb_4bit_use_double_quant=False,
+ bnb_4bit_quant_storage=None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.BITS_AND_BYTES
+
+ if load_in_4bit and load_in_8bit:
+ raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")
+
+ self._load_in_8bit = load_in_8bit
+ self._load_in_4bit = load_in_4bit
+ self.llm_int8_threshold = llm_int8_threshold
+ self.llm_int8_skip_modules = llm_int8_skip_modules
+ self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload
+ self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight
+ self.bnb_4bit_quant_type = bnb_4bit_quant_type
+ self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant
+
+ if bnb_4bit_compute_dtype is None:
+ self.bnb_4bit_compute_dtype = torch.float32
+ elif isinstance(bnb_4bit_compute_dtype, str):
+ self.bnb_4bit_compute_dtype = getattr(torch, bnb_4bit_compute_dtype)
+ elif isinstance(bnb_4bit_compute_dtype, torch.dtype):
+ self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype
+ else:
+ raise ValueError("bnb_4bit_compute_dtype must be a string or a torch.dtype")
+
+ if bnb_4bit_quant_storage is None:
+ self.bnb_4bit_quant_storage = torch.uint8
+ elif isinstance(bnb_4bit_quant_storage, str):
+ if bnb_4bit_quant_storage not in ["float16", "float32", "int8", "uint8", "float64", "bfloat16"]:
+ raise ValueError(
+ "`bnb_4bit_quant_storage` must be a valid string (one of 'float16', 'float32', 'int8', 'uint8', 'float64', 'bfloat16') "
+ )
+ self.bnb_4bit_quant_storage = getattr(torch, bnb_4bit_quant_storage)
+ elif isinstance(bnb_4bit_quant_storage, torch.dtype):
+ self.bnb_4bit_quant_storage = bnb_4bit_quant_storage
+ else:
+ raise ValueError("bnb_4bit_quant_storage must be a string or a torch.dtype")
+
+ if kwargs:
+ logger.info(f"Unused kwargs: {list(kwargs.keys())}. These kwargs are not used in {self.__class__}.")
+
+ self.post_init()
+
+ @property
+ def load_in_4bit(self):
+ return self._load_in_4bit
+
+ @load_in_4bit.setter
+ def load_in_4bit(self, value: bool):
+ if not isinstance(value, bool):
+ raise TypeError("load_in_4bit must be a boolean")
+
+ if self.load_in_8bit and value:
+ raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")
+ self._load_in_4bit = value
+
+ @property
+ def load_in_8bit(self):
+ return self._load_in_8bit
+
+ @load_in_8bit.setter
+ def load_in_8bit(self, value: bool):
+ if not isinstance(value, bool):
+ raise TypeError("load_in_8bit must be a boolean")
+
+ if self.load_in_4bit and value:
+ raise ValueError("load_in_4bit and load_in_8bit are both True, but only one can be used at the same time")
+ self._load_in_8bit = value
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ if not isinstance(self.load_in_4bit, bool):
+ raise TypeError("load_in_4bit must be a boolean")
+
+ if not isinstance(self.load_in_8bit, bool):
+ raise TypeError("load_in_8bit must be a boolean")
+
+ if not isinstance(self.llm_int8_threshold, float):
+ raise TypeError("llm_int8_threshold must be a float")
+
+ if self.llm_int8_skip_modules is not None and not isinstance(self.llm_int8_skip_modules, list):
+ raise TypeError("llm_int8_skip_modules must be a list of strings")
+ if not isinstance(self.llm_int8_enable_fp32_cpu_offload, bool):
+ raise TypeError("llm_int8_enable_fp32_cpu_offload must be a boolean")
+
+ if not isinstance(self.llm_int8_has_fp16_weight, bool):
+ raise TypeError("llm_int8_has_fp16_weight must be a boolean")
+
+ if self.bnb_4bit_compute_dtype is not None and not isinstance(self.bnb_4bit_compute_dtype, torch.dtype):
+ raise TypeError("bnb_4bit_compute_dtype must be torch.dtype")
+
+ if not isinstance(self.bnb_4bit_quant_type, str):
+ raise TypeError("bnb_4bit_quant_type must be a string")
+
+ if not isinstance(self.bnb_4bit_use_double_quant, bool):
+ raise TypeError("bnb_4bit_use_double_quant must be a boolean")
+
+ if self.load_in_4bit and not version.parse(importlib.metadata.version("bitsandbytes")) >= version.parse(
+ "0.39.0"
+ ):
+ raise ValueError(
+ "4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"
+ )
+
+ def is_quantizable(self):
+ r"""
+ Returns `True` if the model is quantizable, `False` otherwise.
+ """
+ return self.load_in_8bit or self.load_in_4bit
+
+ def quantization_method(self):
+ r"""
+ This method returns the quantization method used for the model. If the model is not quantizable, it returns
+ `None`.
+ """
+ if self.load_in_8bit:
+ return "llm_int8"
+ elif self.load_in_4bit and self.bnb_4bit_quant_type == "fp4":
+ return "fp4"
+ elif self.load_in_4bit and self.bnb_4bit_quant_type == "nf4":
+ return "nf4"
+ else:
+ return None
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary. Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+ output = copy.deepcopy(self.__dict__)
+ output["bnb_4bit_compute_dtype"] = str(output["bnb_4bit_compute_dtype"]).split(".")[1]
+ output["bnb_4bit_quant_storage"] = str(output["bnb_4bit_quant_storage"]).split(".")[1]
+ output["load_in_4bit"] = self.load_in_4bit
+ output["load_in_8bit"] = self.load_in_8bit
+
+ return output
+
+ def __repr__(self):
+ config_dict = self.to_dict()
+ return f"{self.__class__.__name__} {json.dumps(config_dict, indent=2, sort_keys=True)}\n"
+
+ def to_diff_dict(self) -> dict[str, Any]:
+ """
+ Removes all attributes from config which correspond to the default config attributes for better readability and
+ serializes to a Python dictionary.
+
+ Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ config_dict = self.to_dict()
+
+ # get the default config dict
+ default_config_dict = BitsAndBytesConfig().to_dict()
+
+ serializable_config_dict = {}
+
+ # only serialize values that differ from the default config
+ for key, value in config_dict.items():
+ if value != default_config_dict[key]:
+ serializable_config_dict[key] = value
+
+ return serializable_config_dict
+
+
+class ExllamaVersion(int, Enum):
+ ONE = 1
+ TWO = 2
+
+
+@dataclass
+class GPTQConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using `optimum` api for gptq quantization relying on auto_gptq backend.
+
+ Args:
+ bits (`int`):
+ The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
+ tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):
+ The tokenizer used to process the dataset. You can pass either:
+ - A custom tokenizer object.
+ - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
+ - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
+ using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
+ dataset (`Union[list[str]]`, *optional*):
+ The dataset used for quantization. You can provide your own dataset in a list of string or just use the
+ original datasets used in GPTQ paper ['wikitext2','c4','c4-new']
+ group_size (`int`, *optional*, defaults to 128):
+ The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.
+ damp_percent (`float`, *optional*, defaults to 0.1):
+ The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.1.
+ desc_act (`bool`, *optional*, defaults to `False`):
+ Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly
+ speed up inference but the perplexity may become slightly worse. Also known as act-order.
+ sym (`bool`, *optional*, defaults to `True`):
+ Whether to use symmetric quantization.
+ true_sequential (`bool`, *optional*, defaults to `True`):
+ Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing
+ the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes
+ quantization using inputs that have passed through the previously quantized layers.
+ checkpoint_format (`str`, *optional*, defaults to `"gptq"`):
+ GPTQ weight format. `gptq`(v1) is supported by both gptqmodel and auto-gptq. `gptq_v2` is gptqmodel only.
+ meta (`dict[str, any]`, *optional*):
+ Properties, such as tooling:version, that do not directly contributes to quantization or quant inference are stored in meta.
+ i.e. `meta.quantizer`: ["optimum:_version_", "gptqmodel:_version_"]
+ backend (`str`, *optional*):
+ Controls which gptq kernel to be used. Valid values for gptqmodel are `auto`, `auto_trainable` and more. For auto-gptq, only
+ valid value is None and `auto_trainable`. Ref gptqmodel backends: https://github.com/ModelCloud/GPTQModel/blob/main/gptqmodel/utils/backend.py
+ use_cuda_fp16 (`bool`, *optional*, defaults to `False`):
+ Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. Auto-gptq only.
+ model_seqlen (`int`, *optional*):
+ The maximum sequence length that the model can take.
+ block_name_to_quantize (`str`, *optional*):
+ The transformers block name to quantize. If None, we will infer the block name using common patterns (e.g. model.layers)
+ module_name_preceding_first_block (`list[str]`, *optional*):
+ The layers that are preceding the first Transformer block.
+ batch_size (`int`, *optional*, defaults to 1):
+ The batch size used when processing the dataset
+ pad_token_id (`int`, *optional*):
+ The pad token id. Needed to prepare the dataset when `batch_size` > 1.
+ use_exllama (`bool`, *optional*):
+ Whether to use exllama backend. Defaults to `True` if unset. Only works with `bits` = 4.
+ max_input_length (`int`, *optional*):
+ The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input
+ length. It is specific to the exllama backend with act-order.
+ exllama_config (`dict[str, Any]`, *optional*):
+ The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults
+ to `{"version": 1}` if unset.
+ cache_block_outputs (`bool`, *optional*, defaults to `True`):
+ Whether to cache block outputs to reuse as inputs for the succeeding block.
+ modules_in_block_to_quantize (`list[list[str]]`, *optional*):
+ List of list of module names to quantize in the specified block. This argument is useful to exclude certain linear modules from being quantized.
+ The block to quantize can be specified by setting `block_name_to_quantize`. We will quantize each list sequentially. If not set, we will quantize all linear layers.
+ Example: `modules_in_block_to_quantize =[["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj"], ["self_attn.o_proj"]]`.
+ In this example, we will first quantize the q,k,v layers simultaneously since they are independent.
+ Then, we will quantize `self_attn.o_proj` layer with the q,k,v layers quantized. This way, we will get
+ better results since it reflects the real input `self_attn.o_proj` will get when the model is quantized.
+ """
+
+ def __init__(
+ self,
+ bits: int,
+ tokenizer: Any = None,
+ dataset: Optional[Union[list[str], str]] = None,
+ group_size: int = 128,
+ damp_percent: float = 0.1,
+ desc_act: bool = False,
+ sym: bool = True,
+ true_sequential: bool = True,
+ checkpoint_format: str = "gptq",
+ meta: Optional[dict[str, Any]] = None,
+ backend: Optional[str] = None,
+ use_cuda_fp16: bool = False,
+ model_seqlen: Optional[int] = None,
+ block_name_to_quantize: Optional[str] = None,
+ module_name_preceding_first_block: Optional[list[str]] = None,
+ batch_size: int = 1,
+ pad_token_id: Optional[int] = None,
+ use_exllama: Optional[bool] = None,
+ max_input_length: Optional[int] = None,
+ exllama_config: Optional[dict[str, Any]] = None,
+ cache_block_outputs: bool = True,
+ modules_in_block_to_quantize: Optional[list[list[str]]] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.GPTQ
+ self.bits = bits
+ self.tokenizer = tokenizer
+ self.dataset = dataset
+ self.group_size = group_size
+ self.damp_percent = damp_percent
+ self.desc_act = desc_act
+ self.sym = sym
+ self.true_sequential = true_sequential
+ self.checkpoint_format = checkpoint_format.lower()
+ self.meta = meta
+ self.backend = backend.lower() if isinstance(backend, str) else backend
+ self.use_cuda_fp16 = use_cuda_fp16
+ self.model_seqlen = model_seqlen
+ self.block_name_to_quantize = block_name_to_quantize
+ self.module_name_preceding_first_block = module_name_preceding_first_block
+ self.batch_size = batch_size
+ self.pad_token_id = pad_token_id
+ self.use_exllama = use_exllama
+ self.max_input_length = max_input_length
+ self.exllama_config = exllama_config
+ self.cache_block_outputs = cache_block_outputs
+ self.modules_in_block_to_quantize = modules_in_block_to_quantize
+ self.post_init()
+
+ def get_loading_attributes(self):
+ attributes_dict = copy.deepcopy(self.__dict__)
+ loading_attributes = [
+ "use_exllama",
+ "exllama_config",
+ "use_cuda_fp16",
+ "max_input_length",
+ "backend",
+ ]
+ loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
+ return loading_attributes_dict
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ if self.bits not in [2, 3, 4, 8]:
+ raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")
+ if self.group_size != -1 and self.group_size <= 0:
+ raise ValueError("group_size must be greater than 0 or equal to -1")
+ if not (0 < self.damp_percent < 1):
+ raise ValueError("damp_percent must between 0 and 1.")
+ if self.dataset is not None:
+ if isinstance(self.dataset, str):
+ if self.dataset in ["ptb", "ptb-new"]:
+ raise ValueError(
+ f"""{self.dataset} dataset was deprecated. You can only choose between
+ ['wikitext2','c4','c4-new']"""
+ )
+ if self.dataset not in ["wikitext2", "c4", "c4-new"]:
+ raise ValueError(
+ f"""You have entered a string value for dataset. You can only choose between
+ ['wikitext2','c4','c4-new'], but we found {self.dataset}"""
+ )
+ elif not isinstance(self.dataset, list):
+ raise ValueError(
+ f"""dataset needs to be either a list of string or a value in
+ ['wikitext2','c4','c4-new'], but we found {self.dataset}"""
+ )
+
+ # make sure backend is back/forward compatible with both gptqmodel (full) and auto-gptq (partial)
+ if is_gptqmodel_available():
+ # convert auto-gptq control into gptqmodel backend
+ if self.backend is None:
+ self.backend = "auto_trainable" if self.use_exllama is not None and not self.use_exllama else "auto"
+ else:
+ # convert gptqmodel backend `auto_trainable` into auto-gptq control
+ if self.backend == "auto_trainable":
+ self.use_exllama = False
+
+ # auto-gptq specific kernel control logic
+ if self.use_exllama is None:
+ # New default behaviour
+ self.use_exllama = True
+
+ if self.exllama_config is None:
+ self.exllama_config = {"version": ExllamaVersion.ONE}
+ else:
+ if "version" not in self.exllama_config:
+ raise ValueError("`exllama_config` needs to have a `version` key.")
+ elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:
+ exllama_version = self.exllama_config["version"]
+ raise ValueError(
+ f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"
+ )
+
+ if self.bits == 4 and self.use_exllama:
+ if self.exllama_config["version"] == ExllamaVersion.ONE:
+ logger.info(
+ "You have activated exllama backend. Note that you can get better inference "
+ "speed using exllamav2 kernel by setting `exllama_config`."
+ )
+ elif self.exllama_config["version"] == ExllamaVersion.TWO:
+ if is_auto_gptq_available():
+ optimum_version = version.parse(importlib.metadata.version("optimum"))
+ autogptq_version = version.parse(importlib.metadata.version("auto_gptq"))
+ if optimum_version <= version.parse("1.13.2") or autogptq_version <= version.parse("0.4.2"):
+ raise ValueError(
+ f"You need optimum > 1.13.2 and auto-gptq > 0.4.2 . Make sure to have that version installed - detected version : optimum {optimum_version} and autogptq {autogptq_version}"
+ )
+ if self.modules_in_block_to_quantize is not None:
+ optimum_version = version.parse(importlib.metadata.version("optimum"))
+ if optimum_version < version.parse("1.15.0"):
+ raise ValueError(
+ "You current version of `optimum` does not support `modules_in_block_to_quantize` quantization argument, please upgrade `optimum` package to a version superior than 1.15.0 ."
+ )
+
+ def to_dict(self):
+ config_dict = super().to_dict()
+ config_dict.pop("disable_exllama", None)
+ return config_dict
+
+ def to_dict_optimum(self):
+ """
+ Get compatible dict for optimum gptq config
+ """
+ quant_dict = self.to_dict()
+ # make it compatible with optimum config
+ quant_dict["disable_exllama"] = not self.use_exllama
+ return quant_dict
+
+ @classmethod
+ def from_dict_optimum(cls, config_dict):
+ """
+ Get compatible class with optimum gptq config dict
+ """
+
+ if "disable_exllama" in config_dict:
+ config_dict["use_exllama"] = not config_dict["disable_exllama"]
+ # switch to None to not trigger the warning
+ config_dict.pop("disable_exllama")
+
+ config = cls(**config_dict)
+ return config
+
+
+@dataclass
+class AwqConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using `auto-awq` library awq quantization relying on auto_awq backend.
+
+ Args:
+ bits (`int`, *optional*, defaults to 4):
+ The number of bits to quantize to.
+ group_size (`int`, *optional*, defaults to 128):
+ The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.
+ zero_point (`bool`, *optional*, defaults to `True`):
+ Whether to use zero point quantization.
+ version (`AWQLinearVersion`, *optional*, defaults to `AWQLinearVersion.GEMM`):
+ The version of the quantization algorithm to use. GEMM is better for big batch_size (e.g. >= 8) otherwise,
+ GEMV is better (e.g. < 8 ). GEMM models are compatible with Exllama kernels.
+ backend (`AwqBackendPackingMethod`, *optional*, defaults to `AwqBackendPackingMethod.AUTOAWQ`):
+ The quantization backend. Some models might be quantized using `llm-awq` backend. This is useful for users
+ that quantize their own models using `llm-awq` library.
+ do_fuse (`bool`, *optional*, defaults to `False`):
+ Whether to fuse attention and mlp layers together for faster inference
+ fuse_max_seq_len (`int`, *optional*):
+ The Maximum sequence length to generate when using fusing.
+ modules_to_fuse (`dict`, *optional*, default to `None`):
+ Overwrite the natively supported fusing scheme with the one specified by the users.
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
+ Note you cannot quantize directly with transformers, please refer to `AutoAWQ` documentation for quantizing HF models.
+ exllama_config (`dict[str, Any]`, *optional*):
+ You can specify the version of the exllama kernel through the `version` key, the maximum sequence
+ length through the `max_input_len` key, and the maximum batch size through the `max_batch_size` key.
+ Defaults to `{"version": 2, "max_input_len": 2048, "max_batch_size": 8}` if unset.
+ """
+
+ def __init__(
+ self,
+ bits: int = 4,
+ group_size: int = 128,
+ zero_point: bool = True,
+ version: AWQLinearVersion = AWQLinearVersion.GEMM,
+ backend: AwqBackendPackingMethod = AwqBackendPackingMethod.AUTOAWQ,
+ do_fuse: Optional[bool] = None,
+ fuse_max_seq_len: Optional[int] = None,
+ modules_to_fuse: Optional[dict] = None,
+ modules_to_not_convert: Optional[list] = None,
+ exllama_config: Optional[dict[str, int]] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.AWQ
+
+ self.bits = bits
+ self.group_size = group_size
+ self.zero_point = zero_point
+ self.version = version
+ self.backend = backend
+ self.fuse_max_seq_len = fuse_max_seq_len
+ self.modules_to_not_convert = modules_to_not_convert
+ self.exllama_config = exllama_config
+
+ self.modules_to_fuse = modules_to_fuse
+ if do_fuse is None:
+ self.do_fuse = modules_to_fuse is not None and len(modules_to_fuse) > 0
+ else:
+ self.do_fuse = do_fuse
+ self.fuse_max_seq_len = fuse_max_seq_len
+
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ if self.backend not in [AwqBackendPackingMethod.AUTOAWQ, AwqBackendPackingMethod.LLMAWQ]:
+ raise ValueError(
+ f"Only supported quantization backends in {AwqBackendPackingMethod.AUTOAWQ} and {AwqBackendPackingMethod.LLMAWQ} - not recognized backend {self.backend}"
+ )
+
+ self.version = AWQLinearVersion.from_str(self.version)
+ if self.version not in [
+ AWQLinearVersion.GEMM,
+ AWQLinearVersion.GEMV,
+ AWQLinearVersion.EXLLAMA,
+ AWQLinearVersion.IPEX,
+ ]:
+ raise ValueError(
+ f"Only supported versions are in [AWQLinearVersion.GEMM, AWQLinearVersion.GEMV, AWQLinearVersion.EXLLAMA, AWQLinearVersion.IPEX] - not recognized version {self.version}"
+ )
+
+ if self.backend == AwqBackendPackingMethod.LLMAWQ:
+ # Only cuda device can run this function
+ if not (torch.cuda.is_available() or torch.xpu.is_available()):
+ raise ValueError("LLM-AWQ backend is only supported on CUDA and XPU")
+ if torch.cuda.is_available():
+ compute_capability = torch.cuda.get_device_capability()
+ major, minor = compute_capability
+ if major < 8:
+ raise ValueError("LLM-AWQ backend is only supported on CUDA GPUs with compute capability >= 8.0")
+
+ if self.do_fuse and self.fuse_max_seq_len is None:
+ raise ValueError(
+ "You cannot enable fused modules without specifying a `fuse_max_seq_len`, make sure to pass a valid `fuse_max_seq_len` for your usecase"
+ )
+
+ if self.do_fuse:
+ awq_version_supports_fusing = False
+ MIN_AWQ_VERSION = "0.1.7"
+ if is_auto_awq_available():
+ awq_version_supports_fusing = version.parse(importlib.metadata.version("autoawq")) >= version.parse(
+ MIN_AWQ_VERSION
+ )
+
+ if not awq_version_supports_fusing:
+ raise ValueError(
+ f"You current version of `autoawq` does not support module fusing, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+ )
+
+ if self.modules_to_not_convert is not None:
+ awq_version_supports_non_conversion = False
+ MIN_AWQ_VERSION = "0.1.8"
+ if is_auto_awq_available():
+ awq_version_supports_non_conversion = version.parse(
+ importlib.metadata.version("autoawq")
+ ) >= version.parse(MIN_AWQ_VERSION)
+
+ if not awq_version_supports_non_conversion:
+ raise ValueError(
+ f"You current version of `autoawq` does not support module quantization skipping, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+ )
+
+ if self.do_fuse and self.modules_to_fuse is not None:
+ required_keys = [
+ "hidden_size",
+ "num_attention_heads",
+ "num_key_value_heads",
+ "mlp",
+ "attention",
+ "layernorm",
+ "use_alibi",
+ ]
+ if not all(key in self.modules_to_fuse for key in required_keys):
+ raise ValueError(
+ f"Required fields are missing in the fusing mapping, required fields are {required_keys}"
+ )
+
+ if self.version == AWQLinearVersion.EXLLAMA:
+ awq_version_supports_exllama = False
+ MIN_AWQ_VERSION = "0.2.0"
+ if is_auto_awq_available():
+ awq_version_supports_exllama = version.parse(importlib.metadata.version("autoawq")) >= version.parse(
+ MIN_AWQ_VERSION
+ )
+
+ if not awq_version_supports_exllama:
+ raise ValueError(
+ f"You current version of `autoawq` does not support exllama backend, "
+ f"please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+ )
+
+ if self.exllama_config is None:
+ self.exllama_config = {"version": ExllamaVersion.TWO, "max_input_len": 2048, "max_batch_size": 8}
+ else:
+ if "version" not in self.exllama_config:
+ raise ValueError("`exllama_config` needs to have a `version` key.")
+ elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:
+ exllama_version = self.exllama_config["version"]
+ raise ValueError(
+ f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"
+ )
+
+ def get_loading_attributes(self):
+ attributes_dict = copy.deepcopy(self.__dict__)
+ loading_attributes = ["version", "do_fuse", "modules_to_fuse", "fuse_max_seq_len", "exllama_config"]
+ loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
+ return loading_attributes_dict
+
+
+@dataclass
+class AqlmConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about `aqlm` parameters.
+
+ Args:
+ in_group_size (`int`, *optional*, defaults to 8):
+ The group size along the input dimension.
+ out_group_size (`int`, *optional*, defaults to 1):
+ The group size along the output dimension. It's recommended to always use 1.
+ num_codebooks (`int`, *optional*, defaults to 1):
+ Number of codebooks for the Additive Quantization procedure.
+ nbits_per_codebook (`int`, *optional*, defaults to 16):
+ Number of bits encoding a single codebook vector. Codebooks size is 2**nbits_per_codebook.
+ linear_weights_not_to_quantize (`Optional[list[str]]`, *optional*):
+ List of full paths of `nn.Linear` weight parameters that shall not be quantized.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional parameters from which to initialize the configuration object.
+ """
+
+ def __init__(
+ self,
+ in_group_size: int = 8,
+ out_group_size: int = 1,
+ num_codebooks: int = 1,
+ nbits_per_codebook: int = 16,
+ linear_weights_not_to_quantize: Optional[list[str]] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.AQLM
+ self.in_group_size = in_group_size
+ self.out_group_size = out_group_size
+ self.num_codebooks = num_codebooks
+ self.nbits_per_codebook = nbits_per_codebook
+ self.linear_weights_not_to_quantize = linear_weights_not_to_quantize
+
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ if not isinstance(self.in_group_size, int):
+ raise TypeError("in_group_size must be a float")
+ if not isinstance(self.out_group_size, int):
+ raise TypeError("out_group_size must be a float")
+ if not isinstance(self.num_codebooks, int):
+ raise TypeError("num_codebooks must be a float")
+ if not isinstance(self.nbits_per_codebook, int):
+ raise TypeError("nbits_per_codebook must be a float")
+
+ if self.linear_weights_not_to_quantize is not None and not isinstance(
+ self.linear_weights_not_to_quantize, list
+ ):
+ raise ValueError("linear_weights_not_to_quantize must be a list of strings")
+
+ if self.linear_weights_not_to_quantize is None:
+ self.linear_weights_not_to_quantize = []
+
+
+@dataclass
+class VptqLayerConfig(QuantizationConfigMixin):
+ """
+ This is used to explain vptq config params for each layer
+ Args:
+ enable_norm (`bool`, *optional*, defaults to `True`): to control if we have scale/bias for fp-weight
+ enable_perm (`bool`, *optional*, defaults to `True`): to perm input_channel or not
+ group_num (`int`, *optional*, defaults to `1`): how many single groups for vector-quantization
+ group_size (`int`, *optional*, defaults to `-1`): depends on out-features
+ indices_as_float (`bool`, *optional*, defaults to `False`): for Finetuning
+ is_indice_packed (`bool`, *optional*, defaults to `True`): should always be True
+ num_centroids (`list`, *optional*, defaults to `[-1, -1]`): centroid numbers of clusters
+ num_res_centroids (`list`, *optional*, defaults to `[-1, -1]`): ditto for residual
+ outlier_size (`int`, *optional*, defaults to `1`): outliers
+ vector_lens (`list`, *optional*, defaults to `[-1, -1]`): centroid vector length in quantization
+ """
+
+ def __init__(
+ self,
+ enable_norm: bool = True,
+ enable_perm: bool = True,
+ group_num: int = 1,
+ group_size: int = -1,
+ in_features: int = -1,
+ indices_as_float: bool = False,
+ is_indice_packed: bool = True,
+ num_centroids: tuple = [-1, -1],
+ num_res_centroids: tuple = [-1, -1],
+ out_features: int = -1,
+ outlier_size: int = 0,
+ vector_lens: tuple = [-1, -1],
+ **kwargs,
+ ):
+ self.enable_norm = enable_norm
+ self.enable_perm = enable_perm
+ self.group_num = group_num
+ self.group_size = group_size
+ self.in_features = in_features
+ self.indices_as_float = indices_as_float
+ self.is_indice_packed = is_indice_packed
+ self.num_centroids = num_centroids
+ self.num_res_centroids = num_res_centroids
+ self.out_features = out_features
+ self.outlier_size = outlier_size
+ self.vector_lens = vector_lens
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ if self.is_indice_packed is False:
+ raise ValueError("is_indice_packed should always be True")
+
+
+@dataclass
+class VptqConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about `vptq` parameters.
+
+ Args:
+ enable_proxy_error (`bool`, *optional*, defaults to `False`): calculate proxy error for each layer
+ config_for_layers (`Dict`, *optional*, defaults to `{}`): quantization params for each layer
+ shared_layer_config (`Dict`, *optional*, defaults to `{}`): shared quantization params among layers
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional parameters from which to initialize the configuration object.
+ """
+
+ def __init__(
+ self,
+ enable_proxy_error: bool = False,
+ config_for_layers: dict[str, Any] = {},
+ shared_layer_config: dict[str, Any] = {},
+ modules_to_not_convert: Optional[list] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.VPTQ
+ self.enable_proxy_error = enable_proxy_error
+ self.config_for_layers: dict[str, Any] = config_for_layers
+ self.shared_layer_config: dict[str, Any] = shared_layer_config
+ self.modules_to_not_convert = modules_to_not_convert
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ for layer_param in self.config_for_layers.values():
+ VptqLayerConfig(**layer_param)
+ if self.enable_proxy_error is True:
+ raise ValueError("enable_proxy_error should always be False until we support training")
+
+
+@dataclass
+class QuantoConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using `quanto`.
+
+ Args:
+ weights (`str`, *optional*, defaults to `"int8"`):
+ The target dtype for the weights after quantization. Supported values are ("float8","int8","int4","int2")
+ activations (`str`, *optional*):
+ The target dtype for the activations after quantization. Supported values are (None,"int8","float8")
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
+ """
+
+ def __init__(
+ self,
+ weights="int8",
+ activations=None,
+ modules_to_not_convert: Optional[list] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.QUANTO
+ self.weights = weights
+ self.activations = activations
+ self.modules_to_not_convert = modules_to_not_convert
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ accepted_weights = ["float8", "int8", "int4", "int2"]
+ accepted_activations = [None, "int8", "float8"]
+ if self.weights not in accepted_weights:
+ raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights}")
+ if self.activations not in accepted_activations:
+ raise ValueError(f"Only support weights in {accepted_activations} but found {self.activations}")
+
+
+@dataclass
+class EetqConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using `eetq`.
+
+ Args:
+ weights (`str`, *optional*, defaults to `"int8"`):
+ The target dtype for the weights. Supported value is only "int8"
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision.
+ """
+
+ def __init__(
+ self,
+ weights: str = "int8",
+ modules_to_not_convert: Optional[list] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.EETQ
+ self.weights = weights
+ self.modules_to_not_convert = modules_to_not_convert
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ accepted_weights = ["int8"]
+ if self.weights not in accepted_weights:
+ raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights}")
+
+
+class CompressedTensorsConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class that handles compressed-tensors quantization config options.
+ It is a wrapper around `compressed_tensors.QuantizationConfig`
+ Args:
+ config_groups (`typing.dict[str, typing.Union[ForwardRef('QuantizationScheme'), typing.list[str]]]`, *optional*):
+ dictionary mapping group name to a quantization scheme definition
+ format (`str`, *optional*, defaults to `"dense"`):
+ format the model is represented as. Set `run_compressed` True to execute model as the
+ compressed format if not `dense`
+ quantization_status (`QuantizationStatus`, *optional*, defaults to `"initialized"`):
+ status of model in the quantization lifecycle, ie 'initialized', 'calibration', 'frozen'
+ kv_cache_scheme (`typing.Union[QuantizationArgs, NoneType]`, *optional*):
+ specifies quantization of the kv cache. If None, kv cache is not quantized.
+ global_compression_ratio (`typing.Union[float, NoneType]`, *optional*):
+ 0-1 float percentage of model compression
+ ignore (`typing.Union[typing.list[str], NoneType]`, *optional*):
+ layer names or types to not quantize, supports regex prefixed by 're:'
+ sparsity_config (`typing.dict[str, typing.Any]`, *optional*):
+ configuration for sparsity compression
+ quant_method (`str`, *optional*, defaults to `"compressed-tensors"`):
+ do not override, should be compressed-tensors
+ run_compressed (`bool`, *optional*, defaults to `True`): alter submodules (usually linear) in order to
+ emulate compressed model execution if True, otherwise use default submodule
+ """
+
+ def __init__(
+ self,
+ config_groups: Optional[dict[str, Union["QuantizationScheme", list[str]]]] = None, # noqa: F821
+ format: str = "dense",
+ quantization_status: "QuantizationStatus" = "initialized", # noqa: F821
+ kv_cache_scheme: Optional["QuantizationArgs"] = None, # noqa: F821
+ global_compression_ratio: Optional[float] = None,
+ ignore: Optional[list[str]] = None,
+ sparsity_config: Optional[dict[str, Any]] = None,
+ quant_method: str = "compressed-tensors",
+ run_compressed: bool = True,
+ **kwargs,
+ ):
+ if is_compressed_tensors_available():
+ from compressed_tensors.config import SparsityCompressionConfig
+ from compressed_tensors.quantization import QuantizationConfig
+ else:
+ raise ImportError(
+ "compressed_tensors is not installed and is required for compressed-tensors quantization. Please install it with `pip install compressed-tensors`."
+ )
+ self.quantization_config = None
+ self.sparsity_config = None
+
+ self.run_compressed = run_compressed
+
+ # parse from dict to load nested QuantizationScheme objects
+ if config_groups or kv_cache_scheme:
+ self.quantization_config = QuantizationConfig.model_validate(
+ {
+ "config_groups": config_groups,
+ "quant_method": quant_method,
+ "format": format,
+ "quantization_status": quantization_status,
+ "kv_cache_scheme": kv_cache_scheme,
+ "global_compression_ratio": global_compression_ratio,
+ "ignore": ignore,
+ **kwargs,
+ }
+ )
+
+ if sparsity_config:
+ self.sparsity_config = SparsityCompressionConfig.load_from_registry(
+ sparsity_config.get("format"), **sparsity_config
+ )
+
+ self.quant_method = QuantizationMethod.COMPRESSED_TENSORS
+
+ def post_init(self):
+ if self.run_compressed:
+ if self.is_sparsification_compressed:
+ logger.warning(
+ "`run_compressed` is only supported for quantized_compressed models"
+ " and not for sparsified models. Setting `run_compressed=False`"
+ )
+ self.run_compressed = False
+ elif not self.is_quantization_compressed:
+ logger.warning(
+ "`run_compressed` is only supported for compressed models. Setting `run_compressed=False`"
+ )
+ self.run_compressed = False
+
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ """
+ Instantiates a [`CompressedTensorsConfig`] from a Python dictionary of parameters.
+ Optionally unwraps any args from the nested quantization_config
+
+ Args:
+ config_dict (`dict[str, Any]`):
+ Dictionary that will be used to instantiate the configuration object.
+ return_unused_kwargs (`bool`,*optional*, defaults to `False`):
+ Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in
+ `PreTrainedModel`.
+ kwargs (`dict[str, Any]`):
+ Additional parameters from which to initialize the configuration object.
+
+ Returns:
+ [`QuantizationConfigMixin`]: The configuration object instantiated from those parameters.
+
+ """
+
+ if "quantization_config" in config_dict:
+ config_dict = dict(
+ sparsity_config=config_dict.get("sparsity_config"),
+ **config_dict["quantization_config"],
+ )
+
+ return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Quantization config to be added to config.json
+
+ Serializes this instance to a Python dictionary. Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+ quantization_config = {}
+ if self.quantization_config is not None:
+ quantization_config = self.quantization_config.model_dump()
+ else:
+ quantization_config["quant_method"] = QuantizationMethod.COMPRESSED_TENSORS
+
+ if self.sparsity_config is not None:
+ quantization_config["sparsity_config"] = self.sparsity_config.model_dump()
+ else:
+ quantization_config["sparsity_config"] = {}
+
+ return quantization_config
+
+ def to_diff_dict(self) -> dict[str, Any]:
+ """
+ Removes all attributes from config which correspond to the default config attributes for better readability and
+ serializes to a Python dictionary.
+ Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
+ """
+ config_dict = self.to_dict()
+
+ # get the default config dict
+ default_config_dict = CompressedTensorsConfig().to_dict()
+
+ serializable_config_dict = {}
+
+ # only serialize values that differ from the default config
+ for key, value in config_dict.items():
+ if key not in default_config_dict or value != default_config_dict[key]:
+ serializable_config_dict[key] = value
+
+ return serializable_config_dict
+
+ def get_loading_attributes(self):
+ return {"run_compressed": self.run_compressed}
+
+ @property
+ def is_quantized(self):
+ return bool(self.quantization_config) and bool(self.quantization_config.config_groups)
+
+ @property
+ def is_quantization_compressed(self):
+ from compressed_tensors.quantization import QuantizationStatus
+
+ return self.is_quantized and self.quantization_config.quantization_status == QuantizationStatus.COMPRESSED
+
+ @property
+ def is_sparsification_compressed(self):
+ from compressed_tensors.config import (
+ CompressionFormat,
+ SparsityCompressionConfig,
+ )
+
+ return (
+ isinstance(self.sparsity_config, SparsityCompressionConfig)
+ and self.sparsity_config.format != CompressionFormat.dense.value
+ )
+
+
+@dataclass
+class FbgemmFp8Config(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using fbgemm fp8 quantization.
+
+ Args:
+ activation_scale_ub (`float`, *optional*, defaults to 1200.0):
+ The activation scale upper bound. This is used when quantizing the input activation.
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision.
+ """
+
+ def __init__(
+ self,
+ activation_scale_ub: float = 1200.0,
+ modules_to_not_convert: Optional[list] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.FBGEMM_FP8
+ self.activation_scale_ub = activation_scale_ub
+ self.modules_to_not_convert = modules_to_not_convert
+
+ def get_loading_attributes(self):
+ attributes_dict = copy.deepcopy(self.__dict__)
+ loading_attributes = ["activation_scale_ub"]
+ loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
+ return loading_attributes_dict
+
+
+@dataclass
+class HiggsConfig(QuantizationConfigMixin):
+ """
+ HiggsConfig is a configuration class for quantization using the HIGGS method.
+
+ Args:
+ bits (int, *optional*, defaults to 4):
+ Number of bits to use for quantization. Can be 2, 3 or 4. Default is 4.
+ p (int, *optional*, defaults to 2):
+ Quantization grid dimension. 1 and 2 are supported. 2 is always better in practice. Default is 2.
+ modules_to_not_convert (`list`, *optional*, default to ["lm_head"]):
+ List of linear layers that should not be quantized.
+ hadamard_size (int, *optional*, defaults to 512):
+ Hadamard size for the HIGGS method. Default is 512. Input dimension of matrices is padded to this value. Decreasing this below 512 will reduce the quality of the quantization.
+ group_size (int, *optional*, defaults to 256):
+ Group size for the HIGGS method. Can be 64, 128 or 256. Decreasing it barely affects the performance. Default is 256. Must be a divisor of hadamard_size.
+ tune_metadata ('dict', *optional*, defaults to {}):
+ Module-wise metadata (gemm block shapes, GPU metadata, etc.) for saving the kernel tuning results. Default is an empty dictionary. Is set automatically during tuning.
+ """
+
+ def __init__(
+ self,
+ bits: int = 4,
+ p: int = 2,
+ modules_to_not_convert: Optional[list[str]] = None,
+ hadamard_size: int = 512,
+ group_size: int = 256,
+ tune_metadata: Optional[dict[str, Any]] = None,
+ **kwargs,
+ ):
+ if tune_metadata is None:
+ tune_metadata = {}
+ self.quant_method = QuantizationMethod.HIGGS
+ self.bits = bits
+ self.p = p
+ self.modules_to_not_convert = modules_to_not_convert
+ self.hadamard_size = hadamard_size
+ self.group_size = group_size
+ self.tune_metadata = tune_metadata
+
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ if self.bits not in [2, 3, 4]:
+ raise ValueError("bits must be 2, 3, or 4")
+ if self.p not in [1, 2]:
+ raise ValueError("p must be 1 or 2. 2 is always better in practice")
+ if self.group_size not in [64, 128, 256]:
+ raise ValueError("group_size must be 64, 128, or 256")
+ if self.hadamard_size % self.group_size != 0:
+ raise ValueError("hadamard_size must be divisible by group_size")
+
+
+@dataclass
+class FPQuantConfig(QuantizationConfigMixin):
+ """
+ FPQuantConfig is a configuration class for quantization using the FPQuant method.
+
+ Args:
+ forward_dtype (`str`, *optional*, defaults to `"mxfp4"`):
+ The dtype to use for the forward pass.
+ forward_method (`str`, *optional*, defaults to `"abs_max"`):
+ The scaling to use for the forward pass. Can be `"abs_max"` or `"quest"`. `"abs_max"` is better for PTQ, `"quest"` is better for QAT.
+ backward_dtype (`str`, *optional*, defaults to `"bf16"`):
+ The dtype to use for the backward pass.
+ store_master_weights (`bool`, *optional*, defaults to `False`):
+ Whether to store the master weights. Needed for QAT over layer weights.
+ hadamard_group_size (`int`, *optional*, defaults to 32):
+ The group size for the hadamard transform before quantization for `"quest"` it matches the MXFP4 group size (32).
+ pseudoquantization (`bool`, *optional*, defaults to `False`):
+ Whether to use Triton-based pseudo-quantization. Is mandatory for non-Blackwell GPUs. Doesn't provide any speedup. For debugging purposes.
+ modules_to_not_convert (`list`, *optional*):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision.
+ """
+
+ def __init__(
+ self,
+ forward_dtype: str = "mxfp4",
+ forward_method: str = "abs_max",
+ backward_dtype: str = "bf16",
+ store_master_weights: bool = False,
+ hadamard_group_size: int = 32,
+ pseudoquantization: bool = False,
+ modules_to_not_convert: Optional[list[str]] = None,
+ **kwargs,
+ ):
+ self.forward_dtype = forward_dtype
+ self.forward_method = forward_method
+ self.backward_dtype = backward_dtype
+ self.store_master_weights = store_master_weights
+ self.hadamard_group_size = hadamard_group_size
+ self.pseudoquantization = pseudoquantization
+ self.modules_to_not_convert = modules_to_not_convert
+
+ self.quant_method = QuantizationMethod.FPQUANT
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ if self.forward_dtype not in ["mxfp4"]:
+ raise ValueError("Only 'mxfp4' is supported for forward_dtype for now.")
+ if self.forward_method not in ["abs_max", "quest"]:
+ raise ValueError("Only 'abs_max' and 'quest' are supported for forward_method for now.")
+ if self.backward_dtype not in ["bf16"]:
+ raise ValueError("Only 'bf16' is supported for backward_dtype for now.")
+ if self.hadamard_group_size not in [32]:
+ raise ValueError("Only a hadamard_group_size of 32 is supported for now.")
+ if self.modules_to_not_convert is None:
+ self.modules_to_not_convert = ["lm_head"]
+
+
+@dataclass
+class TorchAoConfig(QuantizationConfigMixin):
+ quant_method: QuantizationMethod
+ quant_type: Union[str, "AOBaseConfig"] # noqa: F821
+ modules_to_not_convert: Optional[list]
+ quant_type_kwargs: dict[str, Any]
+ include_input_output_embeddings: bool
+ untie_embedding_weights: bool
+
+ """This is a config class for torchao quantization/sparsity techniques.
+
+ Args:
+ quant_type (`Union[str, AOBaseConfig]`):
+ The type of quantization we want to use. Can be either:
+ - A string: currently supporting: `int4_weight_only`, `int8_weight_only` and `int8_dynamic_activation_int8_weight`.
+ - An AOBaseConfig instance: for more advanced configuration options.
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision.
+ include_input_output_embeddings (`bool`, default to `False`):
+ Whether to include embedding in quantization or not, input embedding will be removed from
+ the module_not_to_convert list as well if this flag is set.
+ untie_embedding_weights (`bool`, default to `False`):
+ Whether to untie the weights when we are quantizing input embedding weights that is tied
+ to other weights.
+ kwargs (`dict[str, Any]`, *optional*):
+ The keyword arguments for the chosen type of quantization, for example, int4_weight_only quantization supports two keyword arguments
+ `group_size` and `inner_k_tiles` currently. More API examples and documentation of arguments can be found in
+ https://github.com/pytorch/ao/tree/main/torchao/quantization#other-available-quantization-techniques
+
+ Example:
+
+ ```python
+ # AOBaseConfig-based configuration
+ config = Int4WeightOnlyConfig(group_size=32)
+ quantization_config = TorchAoConfig(config)
+ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", dtype=torch.bfloat16, quantization_config=quantization_config)
+
+ # String-based configuration
+ quantization_config = TorchAoConfig("int4_weight_only", group_size=32)
+ # int4_weight_only quant is only working with *torch.bfloat16* dtype right now
+ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", dtype=torch.bfloat16, quantization_config=quantization_config)
+
+ # autoquant
+ # `autoquant` is a convenient way for users to search for the best quantization for each layer
+ # `min_sqnr` is an option to control the accuracy of the model, higher value means the model is more
+ # accurate, we can start with 30 and adjust it to larger or smaller (e.g. 40, 20)
+ # defaults to None, which means we'll try to get the best performing quantized model without
+ # considering accuracy
+ quantization_config = TorchAoConfig("autoquant", min_sqnr=30)
+ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda", dtype=torch.bfloat16, quantization_config=quantization_config)
+ # run through example inputs, quantization methods will be selected based on the shape of example input
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ input_text = "What are we having for dinner?"
+ input_ids = tokenizer(input_text, return_tensors="pt").to("cuda")
+ MAX_NEW_TOKENS = 1000
+ model.generate(**input_ids, max_new_tokens=MAX_NEW_TOKENS, cache_implementation="static")
+ # manually ran finalize_autoquant if needed
+ if hasattr(quantized_model, "finalize_autoquant"):
+ print("finalizing autoquant")
+ quantized_model.finalize_autoquant()
+
+ ```
+ """
+
+ def __init__(
+ self,
+ quant_type: Union[str, "AOBaseConfig"], # noqa: F821
+ modules_to_not_convert: Optional[list] = None,
+ include_input_output_embeddings: bool = False,
+ untie_embedding_weights: bool = False,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.TORCHAO
+ self.quant_type = quant_type
+ self.modules_to_not_convert = modules_to_not_convert
+ self.quant_type_kwargs = kwargs.get("quant_type_kwargs", kwargs)
+ self.include_input_output_embeddings = include_input_output_embeddings
+ self.untie_embedding_weights = untie_embedding_weights
+ self.post_init()
+
+ @staticmethod
+ def _get_ao_version() -> version.Version:
+ """Centralized check for TorchAO availability and version requirements."""
+ if not is_torchao_available():
+ raise ValueError("TorchAoConfig requires torchao to be installed. Install with `pip install torchao`")
+
+ return version.parse(importlib.metadata.version("torchao"))
+
+ def post_init(self):
+ """Validate configuration and set defaults."""
+ ao_version = self._get_ao_version()
+
+ # Handle quant_type based on type and version
+ if isinstance(self.quant_type, str):
+ self._validate_string_quant_type()
+ elif ao_version > version.parse("0.9.0"):
+ from torchao.quantization.quant_api import AOBaseConfig
+
+ if not isinstance(self.quant_type, AOBaseConfig):
+ raise TypeError(
+ f"quant_type must be either a string or an AOBaseConfig instance, got {type(self.quant_type)}"
+ )
+ else:
+ raise ValueError(
+ f"In torchao <= 0.9.0, quant_type must be a string. Got {type(self.quant_type)}. "
+ f"Please upgrade to torchao > 0.9.0 to use AOBaseConfig instances."
+ )
+
+ def _validate_string_quant_type(self):
+ """Validate string quant_type and its kwargs."""
+ methods = self._get_torchao_quant_type_to_method()
+
+ if self.quant_type not in methods:
+ raise ValueError(
+ f"Unsupported string quantization type: {self.quant_type}. "
+ f"Supported types: {', '.join(methods.keys())}"
+ )
+
+ # Validate kwargs against method signature
+ method = methods[self.quant_type]
+ sig = signature(method)
+ valid_kwargs = {
+ param.name
+ for param in sig.parameters.values()
+ if param.kind in [Parameter.KEYWORD_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
+ }
+
+ invalid_kwargs = set(self.quant_type_kwargs) - valid_kwargs
+ if invalid_kwargs:
+ raise ValueError(
+ f"Unexpected keyword arg for {self.quant_type}: {', '.join(invalid_kwargs)}. "
+ f"Valid kwargs: {', '.join(valid_kwargs)}"
+ )
+
+ def _get_torchao_quant_type_to_method(self):
+ """Get mapping of quant_type strings to their corresponding methods."""
+ from torchao.quantization import (
+ autoquant,
+ int4_weight_only,
+ int8_dynamic_activation_int8_weight,
+ int8_weight_only,
+ )
+
+ return {
+ "int4_weight_only": int4_weight_only,
+ "int8_weight_only": int8_weight_only,
+ "int8_dynamic_activation_int8_weight": int8_dynamic_activation_int8_weight,
+ "autoquant": autoquant,
+ }
+
+ def get_apply_tensor_subclass(self):
+ """Create the appropriate quantization method based on configuration."""
+ if isinstance(self.quant_type, str):
+ methods = self._get_torchao_quant_type_to_method()
+ quant_type_kwargs = self.quant_type_kwargs.copy()
+ if (
+ not torch.cuda.is_available()
+ and is_torchao_available()
+ and self.quant_type == "int4_weight_only"
+ and version.parse(importlib.metadata.version("torchao")) >= version.parse("0.8.0")
+ and quant_type_kwargs.get("layout", None) is None
+ ):
+ if torch.xpu.is_available():
+ if version.parse(importlib.metadata.version("torchao")) >= version.parse(
+ "0.11.0"
+ ) and version.parse(importlib.metadata.version("torch")) > version.parse("2.7.9"):
+ from torchao.dtypes import Int4XPULayout
+ from torchao.quantization.quant_primitives import ZeroPointDomain
+
+ quant_type_kwargs["layout"] = Int4XPULayout()
+ quant_type_kwargs["zero_point_domain"] = ZeroPointDomain.INT
+ else:
+ raise ValueError(
+ "TorchAoConfig requires torchao >= 0.11.0 and torch >= 2.8.0 for XPU support. Please upgrade the version or use run on CPU with the cpu version pytorch."
+ )
+ else:
+ from torchao.dtypes import Int4CPULayout
+
+ quant_type_kwargs["layout"] = Int4CPULayout()
+
+ return methods[self.quant_type](**quant_type_kwargs)
+ else:
+ return self.quant_type
+
+ def to_dict(self):
+ """Convert configuration to a dictionary."""
+ d = super().to_dict()
+
+ if isinstance(self.quant_type, str):
+ # Handle layout serialization if present
+ if "quant_type_kwargs" in d and "layout" in d["quant_type_kwargs"]:
+ if is_dataclass(d["quant_type_kwargs"]["layout"]):
+ d["quant_type_kwargs"]["layout"] = [
+ d["quant_type_kwargs"]["layout"].__class__.__name__,
+ dataclasses.asdict(d["quant_type_kwargs"]["layout"]),
+ ]
+ if isinstance(d["quant_type_kwargs"]["layout"], list):
+ assert len(d["quant_type_kwargs"]["layout"]) == 2, "layout saves layout name and layout kwargs"
+ assert isinstance(d["quant_type_kwargs"]["layout"][0], str), "layout name must be a string"
+ assert isinstance(d["quant_type_kwargs"]["layout"][1], dict), "layout kwargs must be a dict"
+ else:
+ raise ValueError("layout must be a list")
+ else:
+ # Handle AOBaseConfig serialization
+ from torchao.core.config import config_to_dict
+
+ # For now we assume there is 1 config per Transformer, however in the future
+ # We may want to support a config per fqn.
+ d["quant_type"] = {"default": config_to_dict(self.quant_type)}
+
+ return d
+
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ """Create configuration from a dictionary."""
+ ao_version = cls._get_ao_version()
+ assert ao_version > version.parse("0.9.0"), "TorchAoConfig requires torchao > 0.9.0 for construction from dict"
+ config_dict = config_dict.copy()
+ quant_type = config_dict.pop("quant_type")
+
+ if isinstance(quant_type, str):
+ return cls(quant_type=quant_type, **config_dict)
+ # Check if we only have one key which is "default"
+ # In the future we may update this
+ assert len(quant_type) == 1 and "default" in quant_type, (
+ "Expected only one key 'default' in quant_type dictionary"
+ )
+ quant_type = quant_type["default"]
+
+ # Deserialize quant_type if needed
+ from torchao.core.config import config_from_dict
+
+ quant_type = config_from_dict(quant_type)
+
+ return cls(quant_type=quant_type, **config_dict)
+
+
+@dataclass
+class BitNetQuantConfig(QuantizationConfigMixin):
+ """
+ Configuration class for applying BitNet quantization.
+
+ Args:
+ modules_to_not_convert (`Optional[List]`, *optional*):
+ Optionally, provides a list of full paths of `nn.Linear` weight parameters
+ that shall not be quantized. Defaults to None.
+ linear_class (`str`, *optional*, defaults to `"bitlinear"`):
+ The type of linear class to use. Can be either `bitlinear` or `autobitlinear`.
+ quantization_mode (`str`, *optional*, defaults to `"offline"`):
+ The quantization mode to use. Can be either `online` or `offline`.
+ In `online` mode, the weight quantization parameters are calculated dynamically
+ during each forward pass (e.g., based on the current weight values). This can
+ adapt to weight changes during training (Quantization-Aware Training - QAT).
+ In `offline` mode, quantization parameters are pre-calculated *before* inference.
+ These parameters are then fixed and loaded into the quantized model. This
+ generally results in lower runtime overhead compared to online quantization.
+ use_rms_norm (`bool`, *optional*, defaults to `False`):
+ Whether to apply RMSNorm on the activations before quantization. This matches the original BitNet paper's approach
+ of normalizing activations before quantization/packing.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon value used in the RMSNorm layer for numerical stability.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional keyword arguments that may be used by specific quantization
+ backends or future versions.
+ """
+
+ def __init__(
+ self,
+ modules_to_not_convert: Optional[list] = None,
+ linear_class: Optional[str] = "bitlinear",
+ quantization_mode: Optional[str] = "offline",
+ use_rms_norm: Optional[bool] = False,
+ rms_norm_eps: Optional[float] = 1e-6,
+ **kwargs,
+ ):
+ if linear_class not in ["bitlinear", "autobitlinear"]:
+ raise ValueError(f"linear_class must be either 'bitlinear' or 'autobitlinear', but got {linear_class}")
+ if quantization_mode not in ["online", "offline"]:
+ raise ValueError(f"quantization_mode must be either 'online' or 'offline', but got {quantization_mode}")
+ self.quant_method = QuantizationMethod.BITNET
+ self.modules_to_not_convert = modules_to_not_convert
+ self.linear_class = linear_class
+ self.quantization_mode = quantization_mode
+ self.use_rms_norm = use_rms_norm
+ self.rms_norm_eps = rms_norm_eps
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ pass
+
+
+@dataclass
+class SpQRConfig(QuantizationConfigMixin):
+ """
+ This is a wrapper class about `spqr` parameters. Refer to the original publication for more details.
+
+ Args:
+ bits (`int`, *optional*, defaults to 3):
+ Specifies the bit count for the weights and first order zero-points and scales.
+ Currently only bits = 3 is supported.
+ beta1 (`int`, *optional*, defaults to 16):
+ SpQR tile width. Currently only beta1 = 16 is supported.
+ beta2 (`int`, *optional*, defaults to 16):
+ SpQR tile height. Currently only beta2 = 16 is supported.
+ shapes (`Optional`, *optional*):
+ A dictionary holding the shape of each object. We need this because it's impossible
+ to deduce the exact size of the parameters just from bits, beta1, beta2.
+ modules_to_not_convert (`Optional[list[str]]`, *optional*):
+ Optionally, provides a list of full paths of `nn.Linear` weight parameters that shall not be quantized.
+ Defaults to None.
+ kwargs (`dict[str, Any]`, *optional*):
+ Additional parameters from which to initialize the configuration object.
+ """
+
+ def __init__(
+ self,
+ bits: int = 3,
+ beta1: int = 16,
+ beta2: int = 16,
+ shapes: Optional[dict[str, int]] = None,
+ modules_to_not_convert: Optional[list[str]] = None,
+ **kwargs,
+ ):
+ if shapes is None:
+ shapes = {}
+ self.shapes = shapes
+ self.quant_method = QuantizationMethod.SPQR
+ self.bits = bits
+ self.beta1 = beta1
+ self.beta2 = beta2
+ self.modules_to_not_convert = modules_to_not_convert
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
+ """
+ if not isinstance(self.bits, int):
+ raise TypeError("bits must be an int")
+ if not isinstance(self.beta1, int):
+ raise TypeError("beta1 must be an int")
+ if not isinstance(self.beta2, int):
+ raise TypeError("beta2 must be an int")
+
+ if self.bits != 3:
+ raise ValueError("SpQR currently only supports bits = 3")
+ if self.beta1 != 16:
+ raise ValueError("SpQR currently only supports beta1 = 16")
+ if self.beta2 != 16:
+ raise ValueError("SpQR currently only supports beta2 = 16")
+ if not isinstance(self.shapes, dict):
+ raise TypeError("shapes must be a dict")
+
+
+@dataclass
+class FineGrainedFP8Config(QuantizationConfigMixin):
+ """
+ FineGrainedFP8Config is a configuration class for fine-grained FP8 quantization used mainly for deepseek models.
+
+ Args:
+ activation_scheme (`str`, *optional*, defaults to `"dynamic"`):
+ The scheme used for activation, the defaults and only support scheme for now is "dynamic".
+ weight_block_size (`typing.tuple[int, int]`, *optional*, defaults to `(128, 128)`):
+ The size of the weight blocks for quantization, default is (128, 128).
+ modules_to_not_convert (`list`, *optional*):
+ A list of module names that should not be converted during quantization.
+ """
+
+ def __init__(
+ self,
+ activation_scheme: str = "dynamic",
+ weight_block_size: tuple[int, int] = (128, 128),
+ modules_to_not_convert: Optional[list] = None,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.FP8
+ self.modules_to_not_convert = modules_to_not_convert
+ self.activation_scheme = activation_scheme
+ self.weight_block_size = weight_block_size
+ self.post_init()
+
+ def post_init(self):
+ r"""
+ Safety checker that arguments are correct
+ """
+ self.activation_scheme = self.activation_scheme.lower()
+ if self.activation_scheme not in ["dynamic"]:
+ raise ValueError(f"Activation scheme {self.activation_scheme} not supported")
+ if len(self.weight_block_size) != 2:
+ raise ValueError("weight_block_size must be a tuple of two integers")
+ if self.weight_block_size[0] <= 0 or self.weight_block_size[1] <= 0:
+ raise ValueError("weight_block_size must be a tuple of two positive integers")
+
+
+class QuarkConfig(QuantizationConfigMixin):
+ def __init__(
+ self,
+ **kwargs,
+ ):
+ if is_torch_available() and is_quark_available():
+ from quark import __version__ as quark_version
+ from quark.torch.export.config.config import JsonExporterConfig
+ from quark.torch.export.main_export.quant_config_parser import QuantConfigParser
+ from quark.torch.quantization.config.config import Config
+ else:
+ raise ImportError(
+ "Quark is not installed. Please refer to https://quark.docs.amd.com/latest/install.html."
+ )
+ # This might be e.g. `"fp8"` or `"awq"`.
+ self.custom_mode = kwargs["quant_method"]
+ self.legacy = "export" not in kwargs
+
+ if self.custom_mode in ["awq", "fp8"]:
+ # Legacy (quark<1.0) or custom export.
+ self.quant_config = QuantConfigParser.from_custom_config(kwargs, is_bias_quantized=False)
+ self.json_export_config = JsonExporterConfig()
+ else:
+ self.quant_config = Config.from_dict(kwargs)
+
+ if "export" in kwargs:
+ # TODO: Remove this check once configuration version is handled natively by Quark.
+ if "min_kv_scale" in kwargs["export"] and version.parse(quark_version) < version.parse("0.8"):
+ min_kv_scale = kwargs["export"].pop("min_kv_scale")
+ logger.warning(
+ f"The parameter `min_kv_scale={min_kv_scale}` was found in the model config.json's `quantization_config.export` configuration, but this parameter is supported only for quark>=0.8. Ignoring this configuration parameter. Please update the `amd-quark` package."
+ )
+
+ self.json_export_config = JsonExporterConfig(**kwargs["export"])
+ else:
+ # Legacy (quark<1.0) or custom export.
+ self.json_export_config = JsonExporterConfig()
+
+ self.quant_method = QuantizationMethod.QUARK
+
+
+@dataclass
+class Mxfp4Config(QuantizationConfigMixin):
+ """
+ This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded using mxfp4 quantization.
+
+ Args:
+ modules_to_not_convert (`list`, *optional*, default to `None`):
+ The list of modules to not quantize, useful for quantizing models that explicitly require to have
+ some modules left in their original precision.
+ dequantize (`bool`, *optional*, default to `False`):
+ Whether we dequantize the model to bf16 precision or not
+ """
+
+ def __init__(
+ self,
+ modules_to_not_convert: Optional[list] = None,
+ dequantize: bool = False,
+ **kwargs,
+ ):
+ self.quant_method = QuantizationMethod.MXFP4
+ self.modules_to_not_convert = modules_to_not_convert
+ self.dequantize = dequantize
+
+ def get_loading_attributes(self):
+ return {"dequantize": self.dequantize}
+
+ def to_dict(self) -> dict[str, Any]:
+ """
+ Serializes this instance to a Python dictionary. Returns:
+ `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """
+ return {"quant_method": self.quant_method, "modules_to_not_convert": self.modules_to_not_convert}
diff --git a/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2.py b/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f063575fd7af2ae65d3d55b054b856ca8b77fe8
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2.py
@@ -0,0 +1,1511 @@
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: sentencepiece_model.proto
+
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+ name="sentencepiece_model.proto",
+ package="sentencepiece",
+ syntax="proto2",
+ serialized_options=b"H\003",
+ create_key=_descriptor._internal_create_key,
+ serialized_pb=(
+ b'\n\x19sentencepiece_model.proto\x12\rsentencepiece"\xa1\n\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01'
+ b" \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02"
+ b" \x01(\t\x12\x41\n\nmodel_type\x18\x03"
+ b" \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04"
+ b" \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12"
+ b' \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n'
+ b" \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b"
+ b" \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12"
+ b' \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r'
+ b" \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e"
+ b" \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f"
+ b" \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12"
+ b" \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10"
+ b" \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11"
+ b" \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14"
+ b" \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15"
+ b" \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17"
+ b" \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16"
+ b" \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18"
+ b" \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19"
+ b" \x01(\x08:\x05\x66\x61lse\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e"
+ b" \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$"
+ b" \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18"
+ b' \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18"'
+ b" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18)"
+ b" \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+"
+ b" \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05\x12\x16\n\tbos_piece\x18."
+ b" \x01(\t:\x03\x12\x17\n\teos_piece\x18/ \x01(\t:\x04\x12\x18\n\tpad_piece\x18\x30"
+ b" \x01(\t:\x05\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87"
+ b" \x12+\n\x1ctrain_extremely_large_corpus\x18\x31"
+ b' \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01'
+ b" \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03"
+ b" \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12"
+ b" \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06"
+ b' \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01'
+ b' \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01'
+ b" \x01(\t\x12\x10\n\x08\x65xpected\x18\x02"
+ b' \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01'
+ b" \x03(\x0b\x32'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02"
+ b" \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03"
+ b" \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04"
+ b" \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05"
+ b" \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01"
+ b" \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03"
+ b' \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'
+ ),
+)
+
+
+_TRAINERSPEC_MODELTYPE = _descriptor.EnumDescriptor(
+ name="ModelType",
+ full_name="sentencepiece.TrainerSpec.ModelType",
+ filename=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ values=[
+ _descriptor.EnumValueDescriptor(
+ name="UNIGRAM",
+ index=0,
+ number=1,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="BPE",
+ index=1,
+ number=2,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="WORD",
+ index=2,
+ number=3,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="CHAR",
+ index=3,
+ number=4,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ containing_type=None,
+ serialized_options=None,
+ serialized_start=1294,
+ serialized_end=1347,
+)
+_sym_db.RegisterEnumDescriptor(_TRAINERSPEC_MODELTYPE)
+
+_MODELPROTO_SENTENCEPIECE_TYPE = _descriptor.EnumDescriptor(
+ name="Type",
+ full_name="sentencepiece.ModelProto.SentencePiece.Type",
+ filename=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ values=[
+ _descriptor.EnumValueDescriptor(
+ name="NORMAL",
+ index=0,
+ number=1,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="UNKNOWN",
+ index=1,
+ number=2,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="CONTROL",
+ index=2,
+ number=3,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="USER_DEFINED",
+ index=3,
+ number=4,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="BYTE",
+ index=4,
+ number=6,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.EnumValueDescriptor(
+ name="UNUSED",
+ index=5,
+ number=5,
+ serialized_options=None,
+ type=None,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ containing_type=None,
+ serialized_options=None,
+ serialized_start=2100,
+ serialized_end=2184,
+)
+_sym_db.RegisterEnumDescriptor(_MODELPROTO_SENTENCEPIECE_TYPE)
+
+
+_TRAINERSPEC = _descriptor.Descriptor(
+ name="TrainerSpec",
+ full_name="sentencepiece.TrainerSpec",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="input",
+ full_name="sentencepiece.TrainerSpec.input",
+ index=0,
+ number=1,
+ type=9,
+ cpp_type=9,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="input_format",
+ full_name="sentencepiece.TrainerSpec.input_format",
+ index=1,
+ number=7,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="model_prefix",
+ full_name="sentencepiece.TrainerSpec.model_prefix",
+ index=2,
+ number=2,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="model_type",
+ full_name="sentencepiece.TrainerSpec.model_type",
+ index=3,
+ number=3,
+ type=14,
+ cpp_type=8,
+ label=1,
+ has_default_value=True,
+ default_value=1,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="vocab_size",
+ full_name="sentencepiece.TrainerSpec.vocab_size",
+ index=4,
+ number=4,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=8000,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="accept_language",
+ full_name="sentencepiece.TrainerSpec.accept_language",
+ index=5,
+ number=5,
+ type=9,
+ cpp_type=9,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="self_test_sample_size",
+ full_name="sentencepiece.TrainerSpec.self_test_sample_size",
+ index=6,
+ number=6,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=0,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="character_coverage",
+ full_name="sentencepiece.TrainerSpec.character_coverage",
+ index=7,
+ number=10,
+ type=2,
+ cpp_type=6,
+ label=1,
+ has_default_value=True,
+ default_value=0.9995,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="input_sentence_size",
+ full_name="sentencepiece.TrainerSpec.input_sentence_size",
+ index=8,
+ number=11,
+ type=4,
+ cpp_type=4,
+ label=1,
+ has_default_value=True,
+ default_value=0,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="shuffle_input_sentence",
+ full_name="sentencepiece.TrainerSpec.shuffle_input_sentence",
+ index=9,
+ number=19,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="mining_sentence_size",
+ full_name="sentencepiece.TrainerSpec.mining_sentence_size",
+ index=10,
+ number=12,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=False,
+ default_value=0,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=b"\030\001",
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="training_sentence_size",
+ full_name="sentencepiece.TrainerSpec.training_sentence_size",
+ index=11,
+ number=13,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=False,
+ default_value=0,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=b"\030\001",
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="seed_sentencepiece_size",
+ full_name="sentencepiece.TrainerSpec.seed_sentencepiece_size",
+ index=12,
+ number=14,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=1000000,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="shrinking_factor",
+ full_name="sentencepiece.TrainerSpec.shrinking_factor",
+ index=13,
+ number=15,
+ type=2,
+ cpp_type=6,
+ label=1,
+ has_default_value=True,
+ default_value=0.75,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="max_sentence_length",
+ full_name="sentencepiece.TrainerSpec.max_sentence_length",
+ index=14,
+ number=18,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=4192,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="num_threads",
+ full_name="sentencepiece.TrainerSpec.num_threads",
+ index=15,
+ number=16,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=16,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="num_sub_iterations",
+ full_name="sentencepiece.TrainerSpec.num_sub_iterations",
+ index=16,
+ number=17,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=2,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="max_sentencepiece_length",
+ full_name="sentencepiece.TrainerSpec.max_sentencepiece_length",
+ index=17,
+ number=20,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=16,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="split_by_unicode_script",
+ full_name="sentencepiece.TrainerSpec.split_by_unicode_script",
+ index=18,
+ number=21,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="split_by_number",
+ full_name="sentencepiece.TrainerSpec.split_by_number",
+ index=19,
+ number=23,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="split_by_whitespace",
+ full_name="sentencepiece.TrainerSpec.split_by_whitespace",
+ index=20,
+ number=22,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="treat_whitespace_as_suffix",
+ full_name="sentencepiece.TrainerSpec.treat_whitespace_as_suffix",
+ index=21,
+ number=24,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=False,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="split_digits",
+ full_name="sentencepiece.TrainerSpec.split_digits",
+ index=22,
+ number=25,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=False,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="control_symbols",
+ full_name="sentencepiece.TrainerSpec.control_symbols",
+ index=23,
+ number=30,
+ type=9,
+ cpp_type=9,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="user_defined_symbols",
+ full_name="sentencepiece.TrainerSpec.user_defined_symbols",
+ index=24,
+ number=31,
+ type=9,
+ cpp_type=9,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="required_chars",
+ full_name="sentencepiece.TrainerSpec.required_chars",
+ index=25,
+ number=36,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="byte_fallback",
+ full_name="sentencepiece.TrainerSpec.byte_fallback",
+ index=26,
+ number=35,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=False,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="vocabulary_output_piece_score",
+ full_name="sentencepiece.TrainerSpec.vocabulary_output_piece_score",
+ index=27,
+ number=32,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="hard_vocab_limit",
+ full_name="sentencepiece.TrainerSpec.hard_vocab_limit",
+ index=28,
+ number=33,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="use_all_vocab",
+ full_name="sentencepiece.TrainerSpec.use_all_vocab",
+ index=29,
+ number=34,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=False,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="unk_id",
+ full_name="sentencepiece.TrainerSpec.unk_id",
+ index=30,
+ number=40,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=0,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="bos_id",
+ full_name="sentencepiece.TrainerSpec.bos_id",
+ index=31,
+ number=41,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=1,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="eos_id",
+ full_name="sentencepiece.TrainerSpec.eos_id",
+ index=32,
+ number=42,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=2,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="pad_id",
+ full_name="sentencepiece.TrainerSpec.pad_id",
+ index=33,
+ number=43,
+ type=5,
+ cpp_type=1,
+ label=1,
+ has_default_value=True,
+ default_value=-1,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="unk_piece",
+ full_name="sentencepiece.TrainerSpec.unk_piece",
+ index=34,
+ number=45,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=True,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="bos_piece",
+ full_name="sentencepiece.TrainerSpec.bos_piece",
+ index=35,
+ number=46,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=True,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="eos_piece",
+ full_name="sentencepiece.TrainerSpec.eos_piece",
+ index=36,
+ number=47,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=True,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="pad_piece",
+ full_name="sentencepiece.TrainerSpec.pad_piece",
+ index=37,
+ number=48,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=True,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="unk_surface",
+ full_name="sentencepiece.TrainerSpec.unk_surface",
+ index=38,
+ number=44,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=True,
+ default_value=b" \342\201\207 ".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="train_extremely_large_corpus",
+ full_name="sentencepiece.TrainerSpec.train_extremely_large_corpus",
+ index=39,
+ number=49,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=False,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[],
+ enum_types=[
+ _TRAINERSPEC_MODELTYPE,
+ ],
+ serialized_options=None,
+ is_extendable=True,
+ syntax="proto2",
+ extension_ranges=[
+ (200, 536870912),
+ ],
+ oneofs=[],
+ serialized_start=45,
+ serialized_end=1358,
+)
+
+
+_NORMALIZERSPEC = _descriptor.Descriptor(
+ name="NormalizerSpec",
+ full_name="sentencepiece.NormalizerSpec",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="name",
+ full_name="sentencepiece.NormalizerSpec.name",
+ index=0,
+ number=1,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="precompiled_charsmap",
+ full_name="sentencepiece.NormalizerSpec.precompiled_charsmap",
+ index=1,
+ number=2,
+ type=12,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"",
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="add_dummy_prefix",
+ full_name="sentencepiece.NormalizerSpec.add_dummy_prefix",
+ index=2,
+ number=3,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="remove_extra_whitespaces",
+ full_name="sentencepiece.NormalizerSpec.remove_extra_whitespaces",
+ index=3,
+ number=4,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="escape_whitespaces",
+ full_name="sentencepiece.NormalizerSpec.escape_whitespaces",
+ index=4,
+ number=5,
+ type=8,
+ cpp_type=7,
+ label=1,
+ has_default_value=True,
+ default_value=True,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="normalization_rule_tsv",
+ full_name="sentencepiece.NormalizerSpec.normalization_rule_tsv",
+ index=5,
+ number=6,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[],
+ enum_types=[],
+ serialized_options=None,
+ is_extendable=True,
+ syntax="proto2",
+ extension_ranges=[
+ (200, 536870912),
+ ],
+ oneofs=[],
+ serialized_start=1361,
+ serialized_end=1570,
+)
+
+
+_SELFTESTDATA_SAMPLE = _descriptor.Descriptor(
+ name="Sample",
+ full_name="sentencepiece.SelfTestData.Sample",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="input",
+ full_name="sentencepiece.SelfTestData.Sample.input",
+ index=0,
+ number=1,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="expected",
+ full_name="sentencepiece.SelfTestData.Sample.expected",
+ index=1,
+ number=2,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[],
+ enum_types=[],
+ serialized_options=None,
+ is_extendable=False,
+ syntax="proto2",
+ extension_ranges=[],
+ oneofs=[],
+ serialized_start=1641,
+ serialized_end=1682,
+)
+
+_SELFTESTDATA = _descriptor.Descriptor(
+ name="SelfTestData",
+ full_name="sentencepiece.SelfTestData",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="samples",
+ full_name="sentencepiece.SelfTestData.samples",
+ index=0,
+ number=1,
+ type=11,
+ cpp_type=10,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[
+ _SELFTESTDATA_SAMPLE,
+ ],
+ enum_types=[],
+ serialized_options=None,
+ is_extendable=True,
+ syntax="proto2",
+ extension_ranges=[
+ (200, 536870912),
+ ],
+ oneofs=[],
+ serialized_start=1572,
+ serialized_end=1693,
+)
+
+
+_MODELPROTO_SENTENCEPIECE = _descriptor.Descriptor(
+ name="SentencePiece",
+ full_name="sentencepiece.ModelProto.SentencePiece",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="piece",
+ full_name="sentencepiece.ModelProto.SentencePiece.piece",
+ index=0,
+ number=1,
+ type=9,
+ cpp_type=9,
+ label=1,
+ has_default_value=False,
+ default_value=b"".decode("utf-8"),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="score",
+ full_name="sentencepiece.ModelProto.SentencePiece.score",
+ index=1,
+ number=2,
+ type=2,
+ cpp_type=6,
+ label=1,
+ has_default_value=False,
+ default_value=float(0),
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="type",
+ full_name="sentencepiece.ModelProto.SentencePiece.type",
+ index=2,
+ number=3,
+ type=14,
+ cpp_type=8,
+ label=1,
+ has_default_value=True,
+ default_value=1,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[],
+ enum_types=[
+ _MODELPROTO_SENTENCEPIECE_TYPE,
+ ],
+ serialized_options=None,
+ is_extendable=True,
+ syntax="proto2",
+ extension_ranges=[
+ (200, 536870912),
+ ],
+ oneofs=[],
+ serialized_start=1985,
+ serialized_end=2195,
+)
+
+_MODELPROTO = _descriptor.Descriptor(
+ name="ModelProto",
+ full_name="sentencepiece.ModelProto",
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name="pieces",
+ full_name="sentencepiece.ModelProto.pieces",
+ index=0,
+ number=1,
+ type=11,
+ cpp_type=10,
+ label=3,
+ has_default_value=False,
+ default_value=[],
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="trainer_spec",
+ full_name="sentencepiece.ModelProto.trainer_spec",
+ index=1,
+ number=2,
+ type=11,
+ cpp_type=10,
+ label=1,
+ has_default_value=False,
+ default_value=None,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="normalizer_spec",
+ full_name="sentencepiece.ModelProto.normalizer_spec",
+ index=2,
+ number=3,
+ type=11,
+ cpp_type=10,
+ label=1,
+ has_default_value=False,
+ default_value=None,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="self_test_data",
+ full_name="sentencepiece.ModelProto.self_test_data",
+ index=3,
+ number=4,
+ type=11,
+ cpp_type=10,
+ label=1,
+ has_default_value=False,
+ default_value=None,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ _descriptor.FieldDescriptor(
+ name="denormalizer_spec",
+ full_name="sentencepiece.ModelProto.denormalizer_spec",
+ index=4,
+ number=5,
+ type=11,
+ cpp_type=10,
+ label=1,
+ has_default_value=False,
+ default_value=None,
+ message_type=None,
+ enum_type=None,
+ containing_type=None,
+ is_extension=False,
+ extension_scope=None,
+ serialized_options=None,
+ file=DESCRIPTOR,
+ create_key=_descriptor._internal_create_key,
+ ),
+ ],
+ extensions=[],
+ nested_types=[
+ _MODELPROTO_SENTENCEPIECE,
+ ],
+ enum_types=[],
+ serialized_options=None,
+ is_extendable=True,
+ syntax="proto2",
+ extension_ranges=[
+ (200, 536870912),
+ ],
+ oneofs=[],
+ serialized_start=1696,
+ serialized_end=2206,
+)
+
+_TRAINERSPEC.fields_by_name["model_type"].enum_type = _TRAINERSPEC_MODELTYPE
+_TRAINERSPEC_MODELTYPE.containing_type = _TRAINERSPEC
+_SELFTESTDATA_SAMPLE.containing_type = _SELFTESTDATA
+_SELFTESTDATA.fields_by_name["samples"].message_type = _SELFTESTDATA_SAMPLE
+_MODELPROTO_SENTENCEPIECE.fields_by_name["type"].enum_type = _MODELPROTO_SENTENCEPIECE_TYPE
+_MODELPROTO_SENTENCEPIECE.containing_type = _MODELPROTO
+_MODELPROTO_SENTENCEPIECE_TYPE.containing_type = _MODELPROTO_SENTENCEPIECE
+_MODELPROTO.fields_by_name["pieces"].message_type = _MODELPROTO_SENTENCEPIECE
+_MODELPROTO.fields_by_name["trainer_spec"].message_type = _TRAINERSPEC
+_MODELPROTO.fields_by_name["normalizer_spec"].message_type = _NORMALIZERSPEC
+_MODELPROTO.fields_by_name["self_test_data"].message_type = _SELFTESTDATA
+_MODELPROTO.fields_by_name["denormalizer_spec"].message_type = _NORMALIZERSPEC
+DESCRIPTOR.message_types_by_name["TrainerSpec"] = _TRAINERSPEC
+DESCRIPTOR.message_types_by_name["NormalizerSpec"] = _NORMALIZERSPEC
+DESCRIPTOR.message_types_by_name["SelfTestData"] = _SELFTESTDATA
+DESCRIPTOR.message_types_by_name["ModelProto"] = _MODELPROTO
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+TrainerSpec = _reflection.GeneratedProtocolMessageType(
+ "TrainerSpec",
+ (_message.Message,),
+ {
+ "DESCRIPTOR": _TRAINERSPEC,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.TrainerSpec)
+ },
+)
+_sym_db.RegisterMessage(TrainerSpec)
+
+NormalizerSpec = _reflection.GeneratedProtocolMessageType(
+ "NormalizerSpec",
+ (_message.Message,),
+ {
+ "DESCRIPTOR": _NORMALIZERSPEC,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.NormalizerSpec)
+ },
+)
+_sym_db.RegisterMessage(NormalizerSpec)
+
+SelfTestData = _reflection.GeneratedProtocolMessageType(
+ "SelfTestData",
+ (_message.Message,),
+ {
+ "Sample": _reflection.GeneratedProtocolMessageType(
+ "Sample",
+ (_message.Message,),
+ {
+ "DESCRIPTOR": _SELFTESTDATA_SAMPLE,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.SelfTestData.Sample)
+ },
+ ),
+ "DESCRIPTOR": _SELFTESTDATA,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.SelfTestData)
+ },
+)
+_sym_db.RegisterMessage(SelfTestData)
+_sym_db.RegisterMessage(SelfTestData.Sample)
+
+ModelProto = _reflection.GeneratedProtocolMessageType(
+ "ModelProto",
+ (_message.Message,),
+ {
+ "SentencePiece": _reflection.GeneratedProtocolMessageType(
+ "SentencePiece",
+ (_message.Message,),
+ {
+ "DESCRIPTOR": _MODELPROTO_SENTENCEPIECE,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.ModelProto.SentencePiece)
+ },
+ ),
+ "DESCRIPTOR": _MODELPROTO,
+ "__module__": "sentencepiece_model_pb2",
+ # @@protoc_insertion_point(class_scope:sentencepiece.ModelProto)
+ },
+)
+_sym_db.RegisterMessage(ModelProto)
+_sym_db.RegisterMessage(ModelProto.SentencePiece)
+
+
+DESCRIPTOR._options = None
+_TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
+_TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
+# @@protoc_insertion_point(module_scope)
diff --git a/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2_new.py b/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2_new.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ea4f4d64ce660b3d59d60c0650f5849e4fd2251
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/sentencepiece_model_pb2_new.py
@@ -0,0 +1,47 @@
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: sentencepiece_model.proto
+"""Generated protocol buffer code."""
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+ b'\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18 \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05\x12\x16\n\tbos_piece\x18. \x01(\t:\x03\x12\x17\n\teos_piece\x18/ \x01(\t:\x04\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'
+)
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "sentencepiece_model_pb2", _globals)
+if _descriptor._USE_C_DESCRIPTORS is False:
+ DESCRIPTOR._options = None
+ DESCRIPTOR._serialized_options = b"H\003"
+ # (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
+ # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
+ # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
+ # _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
+ # _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
+ _globals["_TRAINERSPEC"]._serialized_start = 45
+ _globals["_TRAINERSPEC"]._serialized_end = 1581
+ _globals["_TRAINERSPEC_MODELTYPE"]._serialized_start = 1517
+ _globals["_TRAINERSPEC_MODELTYPE"]._serialized_end = 1570
+ _globals["_NORMALIZERSPEC"]._serialized_start = 1584
+ _globals["_NORMALIZERSPEC"]._serialized_end = 1793
+ _globals["_SELFTESTDATA"]._serialized_start = 1795
+ _globals["_SELFTESTDATA"]._serialized_end = 1916
+ _globals["_SELFTESTDATA_SAMPLE"]._serialized_start = 1864
+ _globals["_SELFTESTDATA_SAMPLE"]._serialized_end = 1905
+ _globals["_MODELPROTO"]._serialized_start = 1919
+ _globals["_MODELPROTO"]._serialized_end = 2429
+ _globals["_MODELPROTO_SENTENCEPIECE"]._serialized_start = 2208
+ _globals["_MODELPROTO_SENTENCEPIECE"]._serialized_end = 2418
+ _globals["_MODELPROTO_SENTENCEPIECE_TYPE"]._serialized_start = 2323
+ _globals["_MODELPROTO_SENTENCEPIECE_TYPE"]._serialized_end = 2407
+# @@protoc_insertion_point(module_scope)
diff --git a/phivenv/Lib/site-packages/transformers/utils/versions.py b/phivenv/Lib/site-packages/transformers/utils/versions.py
new file mode 100644
index 0000000000000000000000000000000000000000..85dabec8b659022655ce570b4642febd715b6d52
--- /dev/null
+++ b/phivenv/Lib/site-packages/transformers/utils/versions.py
@@ -0,0 +1,117 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Utilities for working with package versions
+"""
+
+import importlib.metadata
+import operator
+import re
+import sys
+from typing import Optional
+
+from packaging import version
+
+
+ops = {
+ "<": operator.lt,
+ "<=": operator.le,
+ "==": operator.eq,
+ "!=": operator.ne,
+ ">=": operator.ge,
+ ">": operator.gt,
+}
+
+
+def _compare_versions(op, got_ver, want_ver, requirement, pkg, hint):
+ if got_ver is None or want_ver is None:
+ raise ValueError(
+ f"Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider"
+ f" reinstalling {pkg}."
+ )
+ if not ops[op](version.parse(got_ver), version.parse(want_ver)):
+ raise ImportError(
+ f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}"
+ )
+
+
+def require_version(requirement: str, hint: Optional[str] = None) -> None:
+ """
+ Perform a runtime check of the dependency versions, using the exact same syntax used by pip.
+
+ The installed module version comes from the *site-packages* dir via *importlib.metadata*.
+
+ Args:
+ requirement (`str`): pip style definition, e.g., "tokenizers==0.9.4", "tqdm>=4.27", "numpy"
+ hint (`str`, *optional*): what suggestion to print in case of requirements not being met
+
+ Example:
+
+ ```python
+ require_version("pandas>1.1.2")
+ require_version("numpy>1.18.5", "this is important to have for whatever reason")
+ ```"""
+
+ hint = f"\n{hint}" if hint is not None else ""
+
+ # non-versioned check
+ if re.match(r"^[\w_\-\d]+$", requirement):
+ pkg, op, want_ver = requirement, None, None
+ else:
+ match = re.findall(r"^([^!=<>\s]+)([\s!=<>]{1,2}.+)", requirement)
+ if not match:
+ raise ValueError(
+ "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but"
+ f" got {requirement}"
+ )
+ pkg, want_full = match[0]
+ want_range = want_full.split(",") # there could be multiple requirements
+ wanted = {}
+ for w in want_range:
+ match = re.findall(r"^([\s!=<>]{1,2})(.+)", w)
+ if not match:
+ raise ValueError(
+ "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,"
+ f" but got {requirement}"
+ )
+ op, want_ver = match[0]
+ wanted[op] = want_ver
+ if op not in ops:
+ raise ValueError(f"{requirement}: need one of {list(ops.keys())}, but got {op}")
+
+ # special case
+ if pkg == "python":
+ got_ver = ".".join([str(x) for x in sys.version_info[:3]])
+ for op, want_ver in wanted.items():
+ _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
+ return
+
+ # check if any version is installed
+ try:
+ got_ver = importlib.metadata.version(pkg)
+ except importlib.metadata.PackageNotFoundError:
+ raise importlib.metadata.PackageNotFoundError(
+ f"The '{requirement}' distribution was not found and is required by this application. {hint}"
+ )
+
+ # check that the right version is installed if version number or a range was provided
+ if want_ver is not None:
+ for op, want_ver in wanted.items():
+ _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
+
+
+def require_version_core(requirement):
+ """require_version wrapper which emits a core-specific hint on failure"""
+ hint = "Try: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main"
+ return require_version(requirement, hint)
diff --git a/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/INSTALLER b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/METADATA b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..b09cb50e1f9a42a8eb58a4339cbdb26250368375
--- /dev/null
+++ b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/METADATA
@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: typing_extensions
+Version: 4.15.0
+Summary: Backported and Experimental Type Hints for Python 3.9+
+Keywords: annotations,backport,checker,checking,function,hinting,hints,type,typechecking,typehinting,typehints,typing
+Author-email: "Guido van Rossum, Jukka Lehtosalo, Łukasz Langa, Michael Lee"
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-Expression: PSF-2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development
+License-File: LICENSE
+Project-URL: Bug Tracker, https://github.com/python/typing_extensions/issues
+Project-URL: Changes, https://github.com/python/typing_extensions/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://typing-extensions.readthedocs.io/
+Project-URL: Home, https://github.com/python/typing_extensions
+Project-URL: Q & A, https://github.com/python/typing/discussions
+Project-URL: Repository, https://github.com/python/typing_extensions
+
+# Typing Extensions
+
+[](https://gitter.im/python/typing)
+
+[Documentation](https://typing-extensions.readthedocs.io/en/latest/#) –
+[PyPI](https://pypi.org/project/typing-extensions/)
+
+## Overview
+
+The `typing_extensions` module serves two related purposes:
+
+- Enable use of new type system features on older Python versions. For example,
+ `typing.TypeGuard` is new in Python 3.10, but `typing_extensions` allows
+ users on previous Python versions to use it too.
+- Enable experimentation with new type system PEPs before they are accepted and
+ added to the `typing` module.
+
+`typing_extensions` is treated specially by static type checkers such as
+mypy and pyright. Objects defined in `typing_extensions` are treated the same
+way as equivalent forms in `typing`.
+
+`typing_extensions` uses
+[Semantic Versioning](https://semver.org/). The
+major version will be incremented only for backwards-incompatible changes.
+Therefore, it's safe to depend
+on `typing_extensions` like this: `typing_extensions ~=x.y`,
+where `x.y` is the first version that includes all features you need.
+[This](https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release)
+is equivalent to `typing_extensions >=x.y, <(x+1)`. Do not depend on `~= x.y.z`
+unless you really know what you're doing; that defeats the purpose of
+semantic versioning.
+
+## Included items
+
+See [the documentation](https://typing-extensions.readthedocs.io/en/latest/#) for a
+complete listing of module contents.
+
+## Contributing
+
+See [CONTRIBUTING.md](https://github.com/python/typing_extensions/blob/main/CONTRIBUTING.md)
+for how to contribute to `typing_extensions`.
+
diff --git a/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/RECORD b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..cbb2e0ac7901b06778cccc7c5471e421c48b7442
--- /dev/null
+++ b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/RECORD
@@ -0,0 +1,7 @@
+__pycache__/typing_extensions.cpython-39.pyc,,
+typing_extensions-4.15.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+typing_extensions-4.15.0.dist-info/METADATA,sha256=wTg3j-jxiTSsmd4GBTXFPsbBOu7WXpTDJkHafuMZKnI,3259
+typing_extensions-4.15.0.dist-info/RECORD,,
+typing_extensions-4.15.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+typing_extensions-4.15.0.dist-info/licenses/LICENSE,sha256=Oy-B_iHRgcSZxZolbI4ZaEVdZonSaaqFNzv7avQdo78,13936
+typing_extensions.py,sha256=Qz0R0XDTok0usGXrwb_oSM6n49fOaFZ6tSvqLUwvftg,160429
diff --git a/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/WHEEL b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37
--- /dev/null
+++ b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any
diff --git a/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/licenses/LICENSE b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..f26bcf4d2de6eb136e31006ca3ab447d5e488adf
--- /dev/null
+++ b/phivenv/Lib/site-packages/typing_extensions-4.15.0.dist-info/licenses/LICENSE
@@ -0,0 +1,279 @@
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC. Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see https://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team. In October of the same
+year, the PythonLabs team moved to Digital Creations, which became
+Zope Corporation. In 2001, the Python Software Foundation (PSF, see
+https://www.python.org/psf/) was formed, a non-profit organization
+created specifically to own Python-related Intellectual Property.
+Zope Corporation was a sponsoring member of the PSF.
+
+All Python releases are Open Source (see https://opensource.org for
+the Open Source Definition). Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+ Release Derived Year Owner GPL-
+ from compatible? (1)
+
+ 0.9.0 thru 1.2 1991-1995 CWI yes
+ 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes
+ 1.6 1.5.2 2000 CNRI no
+ 2.0 1.6 2000 BeOpen.com no
+ 1.6.1 1.6 2001 CNRI yes (2)
+ 2.1 2.0+1.6.1 2001 PSF no
+ 2.0.1 2.0+1.6.1 2001 PSF yes
+ 2.1.1 2.1+2.0.1 2001 PSF yes
+ 2.1.2 2.1.1 2002 PSF yes
+ 2.1.3 2.1.2 2002 PSF yes
+ 2.2 and above 2.1.1 2001-now PSF yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+ the GPL. All Python licenses, unlike the GPL, let you distribute
+ a modified version without making your changes open source. The
+ GPL-compatible licenses make it possible to combine Python with
+ other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+ because its license has a choice of law clause. According to
+ CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+ is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+Python software and documentation are licensed under the
+Python Software Foundation License Version 2.
+
+Starting with Python 3.8.6, examples, recipes, and other code in
+the documentation are dual licensed under the PSF License Version 2
+and the Zero-Clause BSD license.
+
+Some software incorporated into Python is under different licenses.
+The licenses are listed with code falling under that license.
+
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation;
+All Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee. This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions. Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee. This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party. As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee. Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement. This Agreement together with
+Python 1.6.1 may be located on the internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013. This
+Agreement may also be obtained from a proxy server on the internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee. This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+ ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands. All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION
+----------------------------------------------------------------------
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
diff --git a/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/INSTALLER b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/METADATA b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..15116c784ed1f8f0f524513d606a26bf6765e923
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/METADATA
@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: urllib3
+Version: 2.5.0
+Summary: HTTP library with thread-safe connection pooling, file post, and more.
+Project-URL: Changelog, https://github.com/urllib3/urllib3/blob/main/CHANGES.rst
+Project-URL: Documentation, https://urllib3.readthedocs.io
+Project-URL: Code, https://github.com/urllib3/urllib3
+Project-URL: Issue tracker, https://github.com/urllib3/urllib3/issues
+Author-email: Andrey Petrov
+Maintainer-email: Seth Michael Larson , Quentin Pradet , Illia Volochii
+License-Expression: MIT
+License-File: LICENSE.txt
+Keywords: filepost,http,httplib,https,pooling,ssl,threadsafe,urllib
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.9
+Provides-Extra: brotli
+Requires-Dist: brotli>=1.0.9; (platform_python_implementation == 'CPython') and extra == 'brotli'
+Requires-Dist: brotlicffi>=0.8.0; (platform_python_implementation != 'CPython') and extra == 'brotli'
+Provides-Extra: h2
+Requires-Dist: h2<5,>=4; extra == 'h2'
+Provides-Extra: socks
+Requires-Dist: pysocks!=1.5.7,<2.0,>=1.5.6; extra == 'socks'
+Provides-Extra: zstd
+Requires-Dist: zstandard>=0.18.0; extra == 'zstd'
+Description-Content-Type: text/markdown
+
+
+
+
+
+
+
+
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+ 
+
+
+urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the
+Python ecosystem already uses urllib3 and you should too.
+urllib3 brings many critical features that are missing from the Python
+standard libraries:
+
+- Thread safety.
+- Connection pooling.
+- Client-side SSL/TLS verification.
+- File uploads with multipart encoding.
+- Helpers for retrying requests and dealing with HTTP redirects.
+- Support for gzip, deflate, brotli, and zstd encoding.
+- Proxy support for HTTP and SOCKS.
+- 100% test coverage.
+
+urllib3 is powerful and easy to use:
+
+```python3
+>>> import urllib3
+>>> resp = urllib3.request("GET", "http://httpbin.org/robots.txt")
+>>> resp.status
+200
+>>> resp.data
+b"User-agent: *\nDisallow: /deny\n"
+```
+
+## Installing
+
+urllib3 can be installed with [pip](https://pip.pypa.io):
+
+```bash
+$ python -m pip install urllib3
+```
+
+Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3):
+
+```bash
+$ git clone https://github.com/urllib3/urllib3.git
+$ cd urllib3
+$ pip install .
+```
+
+
+## Documentation
+
+urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io).
+
+
+## Community
+
+urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and
+collaborating with other contributors. Drop by and say hello 👋
+
+
+## Contributing
+
+urllib3 happily accepts contributions. Please see our
+[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html)
+for some tips on getting started.
+
+
+## Security Disclosures
+
+To report a security vulnerability, please use the
+[Tidelift security contact](https://tidelift.com/security).
+Tidelift will coordinate the fix and disclosure with maintainers.
+
+
+## Maintainers
+
+- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson)
+- [@pquentin](https://github.com/pquentin) (Quentin Pradet)
+- [@illia-v](https://github.com/illia-v) (Illia Volochii)
+- [@theacodes](https://github.com/theacodes) (Thea Flowers)
+- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro)
+- [@lukasa](https://github.com/lukasa) (Cory Benfield)
+- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco)
+- [@shazow](https://github.com/shazow) (Andrey Petrov)
+
+👋
+
+
+## Sponsorship
+
+If your company benefits from this library, please consider [sponsoring its
+development](https://urllib3.readthedocs.io/en/latest/sponsors.html).
+
+
+## For Enterprise
+
+Professional support for urllib3 is available as part of the [Tidelift
+Subscription][1]. Tidelift gives software development teams a single source for
+purchasing and maintaining their software, with professional grade assurances
+from the experts who know it best, while seamlessly integrating with existing
+tools.
+
+[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme
diff --git a/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/RECORD b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..f41657043902d851871e20dc9f47e87da37543b4
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/RECORD
@@ -0,0 +1,79 @@
+urllib3-2.5.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+urllib3-2.5.0.dist-info/METADATA,sha256=maYkTIZt0a-lkEC-hMZWbCBmcGZyJcYOeRk4_nuTrNc,6461
+urllib3-2.5.0.dist-info/RECORD,,
+urllib3-2.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+urllib3-2.5.0.dist-info/licenses/LICENSE.txt,sha256=Ew46ZNX91dCWp1JpRjSn2d8oRGnehuVzIQAmgEHj1oY,1093
+urllib3/__init__.py,sha256=JMo1tg1nIV1AeJ2vENC_Txfl0e5h6Gzl9DGVk1rWRbo,6979
+urllib3/__pycache__/__init__.cpython-39.pyc,,
+urllib3/__pycache__/_base_connection.cpython-39.pyc,,
+urllib3/__pycache__/_collections.cpython-39.pyc,,
+urllib3/__pycache__/_request_methods.cpython-39.pyc,,
+urllib3/__pycache__/_version.cpython-39.pyc,,
+urllib3/__pycache__/connection.cpython-39.pyc,,
+urllib3/__pycache__/connectionpool.cpython-39.pyc,,
+urllib3/__pycache__/exceptions.cpython-39.pyc,,
+urllib3/__pycache__/fields.cpython-39.pyc,,
+urllib3/__pycache__/filepost.cpython-39.pyc,,
+urllib3/__pycache__/poolmanager.cpython-39.pyc,,
+urllib3/__pycache__/response.cpython-39.pyc,,
+urllib3/_base_connection.py,sha256=T1cwH3RhzsrBh6Bz3AOGVDboRsE7veijqZPXXQTR2Rg,5568
+urllib3/_collections.py,sha256=tM7c6J1iKtWZYV_QGYb8-r7Nr1524Dehnsa0Ufh6_mU,17295
+urllib3/_request_methods.py,sha256=gCeF85SO_UU4WoPwYHIoz_tw-eM_EVOkLFp8OFsC7DA,9931
+urllib3/_version.py,sha256=ZlSUkBo_Pd90B6pM0GDO7l2vitQD3QCK3xPR_K0zFJA,511
+urllib3/connection.py,sha256=iP4pgSJtpusXyYlejzNn-gih_wWCxMU-qy6OU1kaapc,42613
+urllib3/connectionpool.py,sha256=ZEhudsa8BIubD2M0XoxBBsjxbsXwMgUScH7oQ9i-j1Y,43371
+urllib3/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+urllib3/contrib/__pycache__/__init__.cpython-39.pyc,,
+urllib3/contrib/__pycache__/pyopenssl.cpython-39.pyc,,
+urllib3/contrib/__pycache__/socks.cpython-39.pyc,,
+urllib3/contrib/emscripten/__init__.py,sha256=u6KNgzjlFZbuAAXa_ybCR7gQ71VJESnF-IIdDA73brw,733
+urllib3/contrib/emscripten/__pycache__/__init__.cpython-39.pyc,,
+urllib3/contrib/emscripten/__pycache__/connection.cpython-39.pyc,,
+urllib3/contrib/emscripten/__pycache__/fetch.cpython-39.pyc,,
+urllib3/contrib/emscripten/__pycache__/request.cpython-39.pyc,,
+urllib3/contrib/emscripten/__pycache__/response.cpython-39.pyc,,
+urllib3/contrib/emscripten/connection.py,sha256=j8DR_flE7hsoFhNfiqHLiaPaCsVbzG44jgahwvsQ52A,8771
+urllib3/contrib/emscripten/emscripten_fetch_worker.js,sha256=CDfYF_9CDobtx2lGidyJ1zjDEvwNT5F-dchmVWXDh0E,3655
+urllib3/contrib/emscripten/fetch.py,sha256=kco06lWoQ-fdFfN51-nzeTywPVBEHg89WIst33H3xcg,23484
+urllib3/contrib/emscripten/request.py,sha256=mL28szy1KvE3NJhWor5jNmarp8gwplDU-7gwGZY5g0Q,566
+urllib3/contrib/emscripten/response.py,sha256=7oVPENYZHuzEGRtG40HonpH5tAIYHsGcHPbJt2Z0U-Y,9507
+urllib3/contrib/pyopenssl.py,sha256=Xp5Ym05VgXGhHa0C4wlutvHxY8SnKSS6WLb2t5Miu0s,19720
+urllib3/contrib/socks.py,sha256=-iardc61GypsJzD6W6yuRS7KVCyfowcQrl_719H7lIM,7549
+urllib3/exceptions.py,sha256=pziumHf0Vwx3z4gvUy7ou8nlM2yIYX0N3l3znEdeF5U,9938
+urllib3/fields.py,sha256=FCf7UULSkf10cuTRUWTQESzxgl1WT8e2aCy3kfyZins,10829
+urllib3/filepost.py,sha256=U8eNZ-mpKKHhrlbHEEiTxxgK16IejhEa7uz42yqA_dI,2388
+urllib3/http2/__init__.py,sha256=xzrASH7R5ANRkPJOot5lGnATOq3KKuyXzI42rcnwmqs,1741
+urllib3/http2/__pycache__/__init__.cpython-39.pyc,,
+urllib3/http2/__pycache__/connection.cpython-39.pyc,,
+urllib3/http2/__pycache__/probe.cpython-39.pyc,,
+urllib3/http2/connection.py,sha256=4DB0DkZEC3yIkhGjUDIHB17wrYCLaL0Ag5bDW2_mGPI,12694
+urllib3/http2/probe.py,sha256=nnAkqbhAakOiF75rz7W0udZ38Eeh_uD8fjV74N73FEI,3014
+urllib3/poolmanager.py,sha256=oKsgP1EsAI4OVgK9-9D3AYXZS5HYV8yKUSog-QbJ8Ts,23866
+urllib3/py.typed,sha256=UaCuPFa3H8UAakbt-5G8SPacldTOGvJv18pPjUJ5gDY,93
+urllib3/response.py,sha256=TVTSu6Q1U0U7hoHYMIRxxuh4zroeMo8b5EI4DOA13Eo,46480
+urllib3/util/__init__.py,sha256=-qeS0QceivazvBEKDNFCAI-6ACcdDOE4TMvo7SLNlAQ,1001
+urllib3/util/__pycache__/__init__.cpython-39.pyc,,
+urllib3/util/__pycache__/connection.cpython-39.pyc,,
+urllib3/util/__pycache__/proxy.cpython-39.pyc,,
+urllib3/util/__pycache__/request.cpython-39.pyc,,
+urllib3/util/__pycache__/response.cpython-39.pyc,,
+urllib3/util/__pycache__/retry.cpython-39.pyc,,
+urllib3/util/__pycache__/ssl_.cpython-39.pyc,,
+urllib3/util/__pycache__/ssl_match_hostname.cpython-39.pyc,,
+urllib3/util/__pycache__/ssltransport.cpython-39.pyc,,
+urllib3/util/__pycache__/timeout.cpython-39.pyc,,
+urllib3/util/__pycache__/url.cpython-39.pyc,,
+urllib3/util/__pycache__/util.cpython-39.pyc,,
+urllib3/util/__pycache__/wait.cpython-39.pyc,,
+urllib3/util/connection.py,sha256=JjO722lzHlzLXPTkr9ZWBdhseXnMVjMSb1DJLVrXSnQ,4444
+urllib3/util/proxy.py,sha256=seP8-Q5B6bB0dMtwPj-YcZZQ30vHuLqRu-tI0JZ2fzs,1148
+urllib3/util/request.py,sha256=XuAsEBT58DAZYUTwpMH5Hr3A1OPoMNvNIYIunbIqbc8,8411
+urllib3/util/response.py,sha256=vQE639uoEhj1vpjEdxu5lNIhJCSUZkd7pqllUI0BZOA,3374
+urllib3/util/retry.py,sha256=bj-2YUqblxLlv8THg5fxww-DM54XCbjgZXIQ71XioCY,18459
+urllib3/util/ssl_.py,sha256=jxnQ3msYkVaokJVWqHNnAVdVtDdidrTHDeyk50gwqaQ,19786
+urllib3/util/ssl_match_hostname.py,sha256=Di7DU7zokoltapT_F0Sj21ffYxwaS_cE5apOtwueeyA,5845
+urllib3/util/ssltransport.py,sha256=Ez4O8pR_vT8dan_FvqBYS6dgDfBXEMfVfrzcdUoWfi4,8847
+urllib3/util/timeout.py,sha256=4eT1FVeZZU7h7mYD1Jq2OXNe4fxekdNvhoWUkZusRpA,10346
+urllib3/util/url.py,sha256=WRh-TMYXosmgp8m8lT4H5spoHw5yUjlcMCfU53AkoAs,15205
+urllib3/util/util.py,sha256=j3lbZK1jPyiwD34T8IgJzdWEZVT-4E-0vYIJi9UjeNA,1146
+urllib3/util/wait.py,sha256=_ph8IrUR3sqPqi0OopQgJUlH4wzkGeM5CiyA7XGGtmI,4423
diff --git a/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/WHEEL b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any
diff --git a/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/licenses/LICENSE.txt b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/licenses/LICENSE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e6183d0276b26c5b87aecccf8d0d5bcd7b1148d4
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3-2.5.0.dist-info/licenses/LICENSE.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2008-2020 Andrey Petrov and contributors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/phivenv/Lib/site-packages/urllib3/__init__.py b/phivenv/Lib/site-packages/urllib3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fe782c8a45bbabcf240f3cac4303ac12b0ec274
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/__init__.py
@@ -0,0 +1,211 @@
+"""
+Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
+"""
+
+from __future__ import annotations
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+import sys
+import typing
+import warnings
+from logging import NullHandler
+
+from . import exceptions
+from ._base_connection import _TYPE_BODY
+from ._collections import HTTPHeaderDict
+from ._version import __version__
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
+from .filepost import _TYPE_FIELDS, encode_multipart_formdata
+from .poolmanager import PoolManager, ProxyManager, proxy_from_url
+from .response import BaseHTTPResponse, HTTPResponse
+from .util.request import make_headers
+from .util.retry import Retry
+from .util.timeout import Timeout
+
+# Ensure that Python is compiled with OpenSSL 1.1.1+
+# If the 'ssl' module isn't available at all that's
+# fine, we only care if the module is available.
+try:
+ import ssl
+except ImportError:
+ pass
+else:
+ if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
+ warnings.warn(
+ "urllib3 v2 only supports OpenSSL 1.1.1+, currently "
+ f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
+ "See: https://github.com/urllib3/urllib3/issues/3020",
+ exceptions.NotOpenSSLWarning,
+ )
+ elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
+ raise ImportError(
+ "urllib3 v2 only supports OpenSSL 1.1.1+, currently "
+ f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
+ "See: https://github.com/urllib3/urllib3/issues/2168"
+ )
+
+__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
+__license__ = "MIT"
+__version__ = __version__
+
+__all__ = (
+ "HTTPConnectionPool",
+ "HTTPHeaderDict",
+ "HTTPSConnectionPool",
+ "PoolManager",
+ "ProxyManager",
+ "HTTPResponse",
+ "Retry",
+ "Timeout",
+ "add_stderr_logger",
+ "connection_from_url",
+ "disable_warnings",
+ "encode_multipart_formdata",
+ "make_headers",
+ "proxy_from_url",
+ "request",
+ "BaseHTTPResponse",
+)
+
+logging.getLogger(__name__).addHandler(NullHandler())
+
+
+def add_stderr_logger(
+ level: int = logging.DEBUG,
+) -> logging.StreamHandler[typing.TextIO]:
+ """
+ Helper for quickly adding a StreamHandler to the logger. Useful for
+ debugging.
+
+ Returns the handler after adding it.
+ """
+ # This method needs to be in this __init__.py to get the __name__ correct
+ # even if urllib3 is vendored within another package.
+ logger = logging.getLogger(__name__)
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+ logger.addHandler(handler)
+ logger.setLevel(level)
+ logger.debug("Added a stderr logging handler to logger: %s", __name__)
+ return handler
+
+
+# ... Clean up.
+del NullHandler
+
+
+# All warning filters *must* be appended unless you're really certain that they
+# shouldn't be: otherwise, it's very hard for users to use most Python
+# mechanisms to silence them.
+# SecurityWarning's always go off by default.
+warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
+# InsecurePlatformWarning's don't vary between requests, so we keep it default.
+warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
+
+
+def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
+ """
+ Helper for quickly disabling all urllib3 warnings.
+ """
+ warnings.simplefilter("ignore", category)
+
+
+_DEFAULT_POOL = PoolManager()
+
+
+def request(
+ method: str,
+ url: str,
+ *,
+ body: _TYPE_BODY | None = None,
+ fields: _TYPE_FIELDS | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ preload_content: bool | None = True,
+ decode_content: bool | None = True,
+ redirect: bool | None = True,
+ retries: Retry | bool | int | None = None,
+ timeout: Timeout | float | int | None = 3,
+ json: typing.Any | None = None,
+) -> BaseHTTPResponse:
+ """
+ A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
+ Therefore, its side effects could be shared across dependencies relying on it.
+ To avoid side effects create a new ``PoolManager`` instance and use it instead.
+ The method does not accept low-level ``**urlopen_kw`` keyword arguments.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param body:
+ Data to send in the request body, either :class:`str`, :class:`bytes`,
+ an iterable of :class:`str`/:class:`bytes`, or a file-like object.
+
+ :param fields:
+ Data to encode and send in the request body.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc.
+
+ :param bool preload_content:
+ If True, the response's body will be preloaded into memory.
+
+ :param bool decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param redirect:
+ If True, automatically handle redirects (status codes 301, 302,
+ 303, 307, 308). Each redirect counts as a retry. Disabling retries
+ will disable redirect, too.
+
+ :param retries:
+ Configure the number of retries to allow before raising a
+ :class:`~urllib3.exceptions.MaxRetryError` exception.
+
+ If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
+ :class:`~urllib3.util.retry.Retry` object for fine-grained control
+ over different types of retries.
+ Pass an integer number to retry connection errors that many times,
+ but no other types of errors. Pass zero to never retry.
+
+ If ``False``, then retries are disabled and any exception is raised
+ immediately. Also, instead of raising a MaxRetryError on redirects,
+ the redirect response will be returned.
+
+ :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
+
+ :param timeout:
+ If specified, overrides the default timeout for this one
+ request. It may be a float (in seconds) or an instance of
+ :class:`urllib3.util.Timeout`.
+
+ :param json:
+ Data to encode and send as JSON with UTF-encoded in the request body.
+ The ``"Content-Type"`` header will be set to ``"application/json"``
+ unless specified otherwise.
+ """
+
+ return _DEFAULT_POOL.request(
+ method,
+ url,
+ body=body,
+ fields=fields,
+ headers=headers,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ redirect=redirect,
+ retries=retries,
+ timeout=timeout,
+ json=json,
+ )
+
+
+if sys.platform == "emscripten":
+ from .contrib.emscripten import inject_into_urllib3 # noqa: 401
+
+ inject_into_urllib3()
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..16395bbfaf8f663819724fa999d66eea318118de
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/_base_connection.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/_base_connection.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d9e6764c05172d87938a12e267adec60829b0c7b
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/_base_connection.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/_collections.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/_collections.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e389afa11a25724835f6afbe1a9658f5fd15474c
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/_collections.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/_request_methods.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/_request_methods.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4d6d5e10b9fd3a609e5516e632d759c2eacc3725
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/_request_methods.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/_version.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/_version.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0665feba0ea4491b6a6bd70d496713ef94adbe71
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/_version.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/connection.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/connection.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3ac5c2fa58dd492ff8e9a16d0fd75227d0970034
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/connection.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/connectionpool.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/connectionpool.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..36192dbdef74d05cbad9979045bcf3ab38eaec26
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/connectionpool.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/exceptions.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/exceptions.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d3701954f04d639722f7b5734966482e51ea5f69
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/exceptions.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/fields.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/fields.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..186c3f3f6438cae4bd7f655d05adadda13558a33
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/fields.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/filepost.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/filepost.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c4a2588e506674f0427e19e4b7b5eac7842f96dc
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/filepost.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/poolmanager.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/poolmanager.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e8726b0ac29a73e35808ca79bf7445618c74cf65
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/poolmanager.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/__pycache__/response.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/__pycache__/response.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1918cd3f51776a2103ddc984109d22c8df58ec4b
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/__pycache__/response.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/_base_connection.py b/phivenv/Lib/site-packages/urllib3/_base_connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..dc0f318c0b380926eed0f4209d395c79963eaf9e
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/_base_connection.py
@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import typing
+
+from .util.connection import _TYPE_SOCKET_OPTIONS
+from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
+from .util.url import Url
+
+_TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], str]
+
+
+class ProxyConfig(typing.NamedTuple):
+ ssl_context: ssl.SSLContext | None
+ use_forwarding_for_https: bool
+ assert_hostname: None | str | typing.Literal[False]
+ assert_fingerprint: str | None
+
+
+class _ResponseOptions(typing.NamedTuple):
+ # TODO: Remove this in favor of a better
+ # HTTP request/response lifecycle tracking.
+ request_method: str
+ request_url: str
+ preload_content: bool
+ decode_content: bool
+ enforce_content_length: bool
+
+
+if typing.TYPE_CHECKING:
+ import ssl
+ from typing import Protocol
+
+ from .response import BaseHTTPResponse
+
+ class BaseHTTPConnection(Protocol):
+ default_port: typing.ClassVar[int]
+ default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
+
+ host: str
+ port: int
+ timeout: None | (
+ float
+ ) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
+ blocksize: int
+ source_address: tuple[str, int] | None
+ socket_options: _TYPE_SOCKET_OPTIONS | None
+
+ proxy: Url | None
+ proxy_config: ProxyConfig | None
+
+ is_verified: bool
+ proxy_is_verified: bool | None
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 8192,
+ socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ ) -> None: ...
+
+ def set_tunnel(
+ self,
+ host: str,
+ port: int | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ scheme: str = "http",
+ ) -> None: ...
+
+ def connect(self) -> None: ...
+
+ def request(
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ # We know *at least* botocore is depending on the order of the
+ # first 3 parameters so to be safe we only mark the later ones
+ # as keyword-only to ensure we have space to extend.
+ *,
+ chunked: bool = False,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ enforce_content_length: bool = True,
+ ) -> None: ...
+
+ def getresponse(self) -> BaseHTTPResponse: ...
+
+ def close(self) -> None: ...
+
+ @property
+ def is_closed(self) -> bool:
+ """Whether the connection either is brand new or has been previously closed.
+ If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
+ properties must be False.
+ """
+
+ @property
+ def is_connected(self) -> bool:
+ """Whether the connection is actively connected to any origin (proxy or target)"""
+
+ @property
+ def has_connected_to_proxy(self) -> bool:
+ """Whether the connection has successfully connected to its proxy.
+ This returns False if no proxy is in use. Used to determine whether
+ errors are coming from the proxy layer or from tunnelling to the target origin.
+ """
+
+ class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
+ default_port: typing.ClassVar[int]
+ default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
+
+ # Certificate verification methods
+ cert_reqs: int | str | None
+ assert_hostname: None | str | typing.Literal[False]
+ assert_fingerprint: str | None
+ ssl_context: ssl.SSLContext | None
+
+ # Trusted CAs
+ ca_certs: str | None
+ ca_cert_dir: str | None
+ ca_cert_data: None | str | bytes
+
+ # TLS version
+ ssl_minimum_version: int | None
+ ssl_maximum_version: int | None
+ ssl_version: int | str | None # Deprecated
+
+ # Client certificates
+ cert_file: str | None
+ key_file: str | None
+ key_password: str | None
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 16384,
+ socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ cert_reqs: int | str | None = None,
+ assert_hostname: None | str | typing.Literal[False] = None,
+ assert_fingerprint: str | None = None,
+ server_hostname: str | None = None,
+ ssl_context: ssl.SSLContext | None = None,
+ ca_certs: str | None = None,
+ ca_cert_dir: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ ssl_minimum_version: int | None = None,
+ ssl_maximum_version: int | None = None,
+ ssl_version: int | str | None = None, # Deprecated
+ cert_file: str | None = None,
+ key_file: str | None = None,
+ key_password: str | None = None,
+ ) -> None: ...
diff --git a/phivenv/Lib/site-packages/urllib3/_collections.py b/phivenv/Lib/site-packages/urllib3/_collections.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b6c1364213b990c716c39d7cc6427cb319cb948
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/_collections.py
@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import typing
+from collections import OrderedDict
+from enum import Enum, auto
+from threading import RLock
+
+if typing.TYPE_CHECKING:
+ # We can only import Protocol if TYPE_CHECKING because it's a development
+ # dependency, and is not available at runtime.
+ from typing import Protocol
+
+ from typing_extensions import Self
+
+ class HasGettableStringKeys(Protocol):
+ def keys(self) -> typing.Iterator[str]: ...
+
+ def __getitem__(self, key: str) -> str: ...
+
+
+__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
+
+
+# Key type
+_KT = typing.TypeVar("_KT")
+# Value type
+_VT = typing.TypeVar("_VT")
+# Default type
+_DT = typing.TypeVar("_DT")
+
+ValidHTTPHeaderSource = typing.Union[
+ "HTTPHeaderDict",
+ typing.Mapping[str, str],
+ typing.Iterable[tuple[str, str]],
+ "HasGettableStringKeys",
+]
+
+
+class _Sentinel(Enum):
+ not_passed = auto()
+
+
+def ensure_can_construct_http_header_dict(
+ potential: object,
+) -> ValidHTTPHeaderSource | None:
+ if isinstance(potential, HTTPHeaderDict):
+ return potential
+ elif isinstance(potential, typing.Mapping):
+ # Full runtime checking of the contents of a Mapping is expensive, so for the
+ # purposes of typechecking, we assume that any Mapping is the right shape.
+ return typing.cast(typing.Mapping[str, str], potential)
+ elif isinstance(potential, typing.Iterable):
+ # Similarly to Mapping, full runtime checking of the contents of an Iterable is
+ # expensive, so for the purposes of typechecking, we assume that any Iterable
+ # is the right shape.
+ return typing.cast(typing.Iterable[tuple[str, str]], potential)
+ elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
+ return typing.cast("HasGettableStringKeys", potential)
+ else:
+ return None
+
+
+class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
+ """
+ Provides a thread-safe dict-like container which maintains up to
+ ``maxsize`` keys while throwing away the least-recently-used keys beyond
+ ``maxsize``.
+
+ :param maxsize:
+ Maximum number of recent elements to retain.
+
+ :param dispose_func:
+ Every time an item is evicted from the container,
+ ``dispose_func(value)`` is called. Callback which will get called
+ """
+
+ _container: typing.OrderedDict[_KT, _VT]
+ _maxsize: int
+ dispose_func: typing.Callable[[_VT], None] | None
+ lock: RLock
+
+ def __init__(
+ self,
+ maxsize: int = 10,
+ dispose_func: typing.Callable[[_VT], None] | None = None,
+ ) -> None:
+ super().__init__()
+ self._maxsize = maxsize
+ self.dispose_func = dispose_func
+ self._container = OrderedDict()
+ self.lock = RLock()
+
+ def __getitem__(self, key: _KT) -> _VT:
+ # Re-insert the item, moving it to the end of the eviction line.
+ with self.lock:
+ item = self._container.pop(key)
+ self._container[key] = item
+ return item
+
+ def __setitem__(self, key: _KT, value: _VT) -> None:
+ evicted_item = None
+ with self.lock:
+ # Possibly evict the existing value of 'key'
+ try:
+ # If the key exists, we'll overwrite it, which won't change the
+ # size of the pool. Because accessing a key should move it to
+ # the end of the eviction line, we pop it out first.
+ evicted_item = key, self._container.pop(key)
+ self._container[key] = value
+ except KeyError:
+ # When the key does not exist, we insert the value first so that
+ # evicting works in all cases, including when self._maxsize is 0
+ self._container[key] = value
+ if len(self._container) > self._maxsize:
+ # If we didn't evict an existing value, and we've hit our maximum
+ # size, then we have to evict the least recently used item from
+ # the beginning of the container.
+ evicted_item = self._container.popitem(last=False)
+
+ # After releasing the lock on the pool, dispose of any evicted value.
+ if evicted_item is not None and self.dispose_func:
+ _, evicted_value = evicted_item
+ self.dispose_func(evicted_value)
+
+ def __delitem__(self, key: _KT) -> None:
+ with self.lock:
+ value = self._container.pop(key)
+
+ if self.dispose_func:
+ self.dispose_func(value)
+
+ def __len__(self) -> int:
+ with self.lock:
+ return len(self._container)
+
+ def __iter__(self) -> typing.NoReturn:
+ raise NotImplementedError(
+ "Iteration over this class is unlikely to be threadsafe."
+ )
+
+ def clear(self) -> None:
+ with self.lock:
+ # Copy pointers to all values, then wipe the mapping
+ values = list(self._container.values())
+ self._container.clear()
+
+ if self.dispose_func:
+ for value in values:
+ self.dispose_func(value)
+
+ def keys(self) -> set[_KT]: # type: ignore[override]
+ with self.lock:
+ return set(self._container.keys())
+
+
+class HTTPHeaderDictItemView(set[tuple[str, str]]):
+ """
+ HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
+ address.
+
+ If we directly try to get an item with a particular name, we will get a string
+ back that is the concatenated version of all the values:
+
+ >>> d['X-Header-Name']
+ 'Value1, Value2, Value3'
+
+ However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
+ these values based on whether combine=True was called when building up the dictionary
+
+ >>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
+ >>> d.add("A", "2", combine=True)
+ >>> d.add("B", "bar")
+ >>> list(d.items())
+ [
+ ('A', '1, 2'),
+ ('B', 'foo'),
+ ('B', 'bar'),
+ ]
+
+ This class conforms to the interface required by the MutableMapping ABC while
+ also giving us the nonstandard iteration behavior we want; items with duplicate
+ keys, ordered by time of first insertion.
+ """
+
+ _headers: HTTPHeaderDict
+
+ def __init__(self, headers: HTTPHeaderDict) -> None:
+ self._headers = headers
+
+ def __len__(self) -> int:
+ return len(list(self._headers.iteritems()))
+
+ def __iter__(self) -> typing.Iterator[tuple[str, str]]:
+ return self._headers.iteritems()
+
+ def __contains__(self, item: object) -> bool:
+ if isinstance(item, tuple) and len(item) == 2:
+ passed_key, passed_val = item
+ if isinstance(passed_key, str) and isinstance(passed_val, str):
+ return self._headers._has_value_for_header(passed_key, passed_val)
+ return False
+
+
+class HTTPHeaderDict(typing.MutableMapping[str, str]):
+ """
+ :param headers:
+ An iterable of field-value pairs. Must not contain multiple field names
+ when compared case-insensitively.
+
+ :param kwargs:
+ Additional field-value pairs to pass in to ``dict.update``.
+
+ A ``dict`` like container for storing HTTP Headers.
+
+ Field names are stored and compared case-insensitively in compliance with
+ RFC 7230. Iteration provides the first case-sensitive key seen for each
+ case-insensitive pair.
+
+ Using ``__setitem__`` syntax overwrites fields that compare equal
+ case-insensitively in order to maintain ``dict``'s api. For fields that
+ compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
+ in a loop.
+
+ If multiple fields that are equal case-insensitively are passed to the
+ constructor or ``.update``, the behavior is undefined and some will be
+ lost.
+
+ >>> headers = HTTPHeaderDict()
+ >>> headers.add('Set-Cookie', 'foo=bar')
+ >>> headers.add('set-cookie', 'baz=quxx')
+ >>> headers['content-length'] = '7'
+ >>> headers['SET-cookie']
+ 'foo=bar, baz=quxx'
+ >>> headers['Content-Length']
+ '7'
+ """
+
+ _container: typing.MutableMapping[str, list[str]]
+
+ def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
+ super().__init__()
+ self._container = {} # 'dict' is insert-ordered
+ if headers is not None:
+ if isinstance(headers, HTTPHeaderDict):
+ self._copy_from(headers)
+ else:
+ self.extend(headers)
+ if kwargs:
+ self.extend(kwargs)
+
+ def __setitem__(self, key: str, val: str) -> None:
+ # avoid a bytes/str comparison by decoding before httplib
+ if isinstance(key, bytes):
+ key = key.decode("latin-1")
+ self._container[key.lower()] = [key, val]
+
+ def __getitem__(self, key: str) -> str:
+ val = self._container[key.lower()]
+ return ", ".join(val[1:])
+
+ def __delitem__(self, key: str) -> None:
+ del self._container[key.lower()]
+
+ def __contains__(self, key: object) -> bool:
+ if isinstance(key, str):
+ return key.lower() in self._container
+ return False
+
+ def setdefault(self, key: str, default: str = "") -> str:
+ return super().setdefault(key, default)
+
+ def __eq__(self, other: object) -> bool:
+ maybe_constructable = ensure_can_construct_http_header_dict(other)
+ if maybe_constructable is None:
+ return False
+ else:
+ other_as_http_header_dict = type(self)(maybe_constructable)
+
+ return {k.lower(): v for k, v in self.itermerged()} == {
+ k.lower(): v for k, v in other_as_http_header_dict.itermerged()
+ }
+
+ def __ne__(self, other: object) -> bool:
+ return not self.__eq__(other)
+
+ def __len__(self) -> int:
+ return len(self._container)
+
+ def __iter__(self) -> typing.Iterator[str]:
+ # Only provide the originally cased names
+ for vals in self._container.values():
+ yield vals[0]
+
+ def discard(self, key: str) -> None:
+ try:
+ del self[key]
+ except KeyError:
+ pass
+
+ def add(self, key: str, val: str, *, combine: bool = False) -> None:
+ """Adds a (name, value) pair, doesn't overwrite the value if it already
+ exists.
+
+ If this is called with combine=True, instead of adding a new header value
+ as a distinct item during iteration, this will instead append the value to
+ any existing header value with a comma. If no existing header value exists
+ for the key, then the value will simply be added, ignoring the combine parameter.
+
+ >>> headers = HTTPHeaderDict(foo='bar')
+ >>> headers.add('Foo', 'baz')
+ >>> headers['foo']
+ 'bar, baz'
+ >>> list(headers.items())
+ [('foo', 'bar'), ('foo', 'baz')]
+ >>> headers.add('foo', 'quz', combine=True)
+ >>> list(headers.items())
+ [('foo', 'bar, baz, quz')]
+ """
+ # avoid a bytes/str comparison by decoding before httplib
+ if isinstance(key, bytes):
+ key = key.decode("latin-1")
+ key_lower = key.lower()
+ new_vals = [key, val]
+ # Keep the common case aka no item present as fast as possible
+ vals = self._container.setdefault(key_lower, new_vals)
+ if new_vals is not vals:
+ # if there are values here, then there is at least the initial
+ # key/value pair
+ assert len(vals) >= 2
+ if combine:
+ vals[-1] = vals[-1] + ", " + val
+ else:
+ vals.append(val)
+
+ def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
+ """Generic import function for any type of header-like object.
+ Adapted version of MutableMapping.update in order to insert items
+ with self.add instead of self.__setitem__
+ """
+ if len(args) > 1:
+ raise TypeError(
+ f"extend() takes at most 1 positional arguments ({len(args)} given)"
+ )
+ other = args[0] if len(args) >= 1 else ()
+
+ if isinstance(other, HTTPHeaderDict):
+ for key, val in other.iteritems():
+ self.add(key, val)
+ elif isinstance(other, typing.Mapping):
+ for key, val in other.items():
+ self.add(key, val)
+ elif isinstance(other, typing.Iterable):
+ other = typing.cast(typing.Iterable[tuple[str, str]], other)
+ for key, value in other:
+ self.add(key, value)
+ elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
+ # THIS IS NOT A TYPESAFE BRANCH
+ # In this branch, the object has a `keys` attr but is not a Mapping or any of
+ # the other types indicated in the method signature. We do some stuff with
+ # it as though it partially implements the Mapping interface, but we're not
+ # doing that stuff safely AT ALL.
+ for key in other.keys():
+ self.add(key, other[key])
+
+ for key, value in kwargs.items():
+ self.add(key, value)
+
+ @typing.overload
+ def getlist(self, key: str) -> list[str]: ...
+
+ @typing.overload
+ def getlist(self, key: str, default: _DT) -> list[str] | _DT: ...
+
+ def getlist(
+ self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
+ ) -> list[str] | _DT:
+ """Returns a list of all the values for the named field. Returns an
+ empty list if the key doesn't exist."""
+ try:
+ vals = self._container[key.lower()]
+ except KeyError:
+ if default is _Sentinel.not_passed:
+ # _DT is unbound; empty list is instance of List[str]
+ return []
+ # _DT is bound; default is instance of _DT
+ return default
+ else:
+ # _DT may or may not be bound; vals[1:] is instance of List[str], which
+ # meets our external interface requirement of `Union[List[str], _DT]`.
+ return vals[1:]
+
+ def _prepare_for_method_change(self) -> Self:
+ """
+ Remove content-specific header fields before changing the request
+ method to GET or HEAD according to RFC 9110, Section 15.4.
+ """
+ content_specific_headers = [
+ "Content-Encoding",
+ "Content-Language",
+ "Content-Location",
+ "Content-Type",
+ "Content-Length",
+ "Digest",
+ "Last-Modified",
+ ]
+ for header in content_specific_headers:
+ self.discard(header)
+ return self
+
+ # Backwards compatibility for httplib
+ getheaders = getlist
+ getallmatchingheaders = getlist
+ iget = getlist
+
+ # Backwards compatibility for http.cookiejar
+ get_all = getlist
+
+ def __repr__(self) -> str:
+ return f"{type(self).__name__}({dict(self.itermerged())})"
+
+ def _copy_from(self, other: HTTPHeaderDict) -> None:
+ for key in other:
+ val = other.getlist(key)
+ self._container[key.lower()] = [key, *val]
+
+ def copy(self) -> Self:
+ clone = type(self)()
+ clone._copy_from(self)
+ return clone
+
+ def iteritems(self) -> typing.Iterator[tuple[str, str]]:
+ """Iterate over all header lines, including duplicate ones."""
+ for key in self:
+ vals = self._container[key.lower()]
+ for val in vals[1:]:
+ yield vals[0], val
+
+ def itermerged(self) -> typing.Iterator[tuple[str, str]]:
+ """Iterate over all headers, merging duplicate ones together."""
+ for key in self:
+ val = self._container[key.lower()]
+ yield val[0], ", ".join(val[1:])
+
+ def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
+ return HTTPHeaderDictItemView(self)
+
+ def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
+ if header_name in self:
+ return potential_value in self._container[header_name.lower()][1:]
+ return False
+
+ def __ior__(self, other: object) -> HTTPHeaderDict:
+ # Supports extending a header dict in-place using operator |=
+ # combining items with add instead of __setitem__
+ maybe_constructable = ensure_can_construct_http_header_dict(other)
+ if maybe_constructable is None:
+ return NotImplemented
+ self.extend(maybe_constructable)
+ return self
+
+ def __or__(self, other: object) -> Self:
+ # Supports merging header dicts using operator |
+ # combining items with add instead of __setitem__
+ maybe_constructable = ensure_can_construct_http_header_dict(other)
+ if maybe_constructable is None:
+ return NotImplemented
+ result = self.copy()
+ result.extend(maybe_constructable)
+ return result
+
+ def __ror__(self, other: object) -> Self:
+ # Supports merging header dicts using operator | when other is on left side
+ # combining items with add instead of __setitem__
+ maybe_constructable = ensure_can_construct_http_header_dict(other)
+ if maybe_constructable is None:
+ return NotImplemented
+ result = type(self)(maybe_constructable)
+ result.extend(self)
+ return result
diff --git a/phivenv/Lib/site-packages/urllib3/_request_methods.py b/phivenv/Lib/site-packages/urllib3/_request_methods.py
new file mode 100644
index 0000000000000000000000000000000000000000..297c271bf401c1cb48c6225f8822e78f58c3ca56
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/_request_methods.py
@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import json as _json
+import typing
+from urllib.parse import urlencode
+
+from ._base_connection import _TYPE_BODY
+from ._collections import HTTPHeaderDict
+from .filepost import _TYPE_FIELDS, encode_multipart_formdata
+from .response import BaseHTTPResponse
+
+__all__ = ["RequestMethods"]
+
+_TYPE_ENCODE_URL_FIELDS = typing.Union[
+ typing.Sequence[tuple[str, typing.Union[str, bytes]]],
+ typing.Mapping[str, typing.Union[str, bytes]],
+]
+
+
+class RequestMethods:
+ """
+ Convenience mixin for classes who implement a :meth:`urlopen` method, such
+ as :class:`urllib3.HTTPConnectionPool` and
+ :class:`urllib3.PoolManager`.
+
+ Provides behavior for making common types of HTTP request methods and
+ decides which type of request field encoding to use.
+
+ Specifically,
+
+ :meth:`.request_encode_url` is for sending requests whose fields are
+ encoded in the URL (such as GET, HEAD, DELETE).
+
+ :meth:`.request_encode_body` is for sending requests whose fields are
+ encoded in the *body* of the request using multipart or www-form-urlencoded
+ (such as for POST, PUT, PATCH).
+
+ :meth:`.request` is for making any kind of request, it will look up the
+ appropriate encoding format and use one of the above two methods to make
+ the request.
+
+ Initializer parameters:
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+ """
+
+ _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
+
+ def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
+ self.headers = headers or {}
+
+ def urlopen(
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ encode_multipart: bool = True,
+ multipart_boundary: str | None = None,
+ **kw: typing.Any,
+ ) -> BaseHTTPResponse: # Abstract
+ raise NotImplementedError(
+ "Classes extending RequestMethods must implement "
+ "their own ``urlopen`` method."
+ )
+
+ def request(
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ fields: _TYPE_FIELDS | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ json: typing.Any | None = None,
+ **urlopen_kw: typing.Any,
+ ) -> BaseHTTPResponse:
+ """
+ Make a request using :meth:`urlopen` with the appropriate encoding of
+ ``fields`` based on the ``method`` used.
+
+ This is a convenience method that requires the least amount of manual
+ effort. It can be used in most situations, while still having the
+ option to drop down to more specific methods when necessary, such as
+ :meth:`request_encode_url`, :meth:`request_encode_body`,
+ or even the lowest level :meth:`urlopen`.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param body:
+ Data to send in the request body, either :class:`str`, :class:`bytes`,
+ an iterable of :class:`str`/:class:`bytes`, or a file-like object.
+
+ :param fields:
+ Data to encode and send in the URL or request body, depending on ``method``.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+
+ :param json:
+ Data to encode and send as JSON with UTF-encoded in the request body.
+ The ``"Content-Type"`` header will be set to ``"application/json"``
+ unless specified otherwise.
+ """
+ method = method.upper()
+
+ if json is not None and body is not None:
+ raise TypeError(
+ "request got values for both 'body' and 'json' parameters which are mutually exclusive"
+ )
+
+ if json is not None:
+ if headers is None:
+ headers = self.headers
+
+ if not ("content-type" in map(str.lower, headers.keys())):
+ headers = HTTPHeaderDict(headers)
+ headers["Content-Type"] = "application/json"
+
+ body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
+ "utf-8"
+ )
+
+ if body is not None:
+ urlopen_kw["body"] = body
+
+ if method in self._encode_url_methods:
+ return self.request_encode_url(
+ method,
+ url,
+ fields=fields, # type: ignore[arg-type]
+ headers=headers,
+ **urlopen_kw,
+ )
+ else:
+ return self.request_encode_body(
+ method, url, fields=fields, headers=headers, **urlopen_kw
+ )
+
+ def request_encode_url(
+ self,
+ method: str,
+ url: str,
+ fields: _TYPE_ENCODE_URL_FIELDS | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ **urlopen_kw: str,
+ ) -> BaseHTTPResponse:
+ """
+ Make a request using :meth:`urlopen` with the ``fields`` encoded in
+ the url. This is useful for request methods like GET, HEAD, DELETE, etc.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param fields:
+ Data to encode and send in the URL.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+ """
+ if headers is None:
+ headers = self.headers
+
+ extra_kw: dict[str, typing.Any] = {"headers": headers}
+ extra_kw.update(urlopen_kw)
+
+ if fields:
+ url += "?" + urlencode(fields)
+
+ return self.urlopen(method, url, **extra_kw)
+
+ def request_encode_body(
+ self,
+ method: str,
+ url: str,
+ fields: _TYPE_FIELDS | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ encode_multipart: bool = True,
+ multipart_boundary: str | None = None,
+ **urlopen_kw: str,
+ ) -> BaseHTTPResponse:
+ """
+ Make a request using :meth:`urlopen` with the ``fields`` encoded in
+ the body. This is useful for request methods like POST, PUT, PATCH, etc.
+
+ When ``encode_multipart=True`` (default), then
+ :func:`urllib3.encode_multipart_formdata` is used to encode
+ the payload with the appropriate content type. Otherwise
+ :func:`urllib.parse.urlencode` is used with the
+ 'application/x-www-form-urlencoded' content type.
+
+ Multipart encoding must be used when posting files, and it's reasonably
+ safe to use it in other times too. However, it may break request
+ signing, such as with OAuth.
+
+ Supports an optional ``fields`` parameter of key/value strings AND
+ key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
+ the MIME type is optional. For example::
+
+ fields = {
+ 'foo': 'bar',
+ 'fakefile': ('foofile.txt', 'contents of foofile'),
+ 'realfile': ('barfile.txt', open('realfile').read()),
+ 'typedfile': ('bazfile.bin', open('bazfile').read(),
+ 'image/jpeg'),
+ 'nonamefile': 'contents of nonamefile field',
+ }
+
+ When uploading a file, providing a filename (the first parameter of the
+ tuple) is optional but recommended to best mimic behavior of browsers.
+
+ Note that if ``headers`` are supplied, the 'Content-Type' header will
+ be overwritten because it depends on the dynamic random boundary string
+ which is used to compose the body of the request. The random boundary
+ string can be explicitly set with the ``multipart_boundary`` parameter.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param fields:
+ Data to encode and send in the request body.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+
+ :param encode_multipart:
+ If True, encode the ``fields`` using the multipart/form-data MIME
+ format.
+
+ :param multipart_boundary:
+ If not specified, then a random boundary will be generated using
+ :func:`urllib3.filepost.choose_boundary`.
+ """
+ if headers is None:
+ headers = self.headers
+
+ extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
+ body: bytes | str
+
+ if fields:
+ if "body" in urlopen_kw:
+ raise TypeError(
+ "request got values for both 'fields' and 'body', can only specify one."
+ )
+
+ if encode_multipart:
+ body, content_type = encode_multipart_formdata(
+ fields, boundary=multipart_boundary
+ )
+ else:
+ body, content_type = (
+ urlencode(fields), # type: ignore[arg-type]
+ "application/x-www-form-urlencoded",
+ )
+
+ extra_kw["body"] = body
+ extra_kw["headers"].setdefault("Content-Type", content_type)
+
+ extra_kw.update(urlopen_kw)
+
+ return self.urlopen(method, url, **extra_kw)
diff --git a/phivenv/Lib/site-packages/urllib3/_version.py b/phivenv/Lib/site-packages/urllib3/_version.py
new file mode 100644
index 0000000000000000000000000000000000000000..49707ce59783e1f6c31fc722c949232506af7dda
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/_version.py
@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+ from typing import Tuple
+ from typing import Union
+
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+ VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '2.5.0'
+__version_tuple__ = version_tuple = (2, 5, 0)
diff --git a/phivenv/Lib/site-packages/urllib3/connection.py b/phivenv/Lib/site-packages/urllib3/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..8082387d94b6559ca2a24126ac1511285389dc6f
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/connection.py
@@ -0,0 +1,1093 @@
+from __future__ import annotations
+
+import datetime
+import http.client
+import logging
+import os
+import re
+import socket
+import sys
+import threading
+import typing
+import warnings
+from http.client import HTTPConnection as _HTTPConnection
+from http.client import HTTPException as HTTPException # noqa: F401
+from http.client import ResponseNotReady
+from socket import timeout as SocketTimeout
+
+if typing.TYPE_CHECKING:
+ from .response import HTTPResponse
+ from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
+ from .util.ssltransport import SSLTransport
+
+from ._collections import HTTPHeaderDict
+from .http2 import probe as http2_probe
+from .util.response import assert_header_parsing
+from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
+from .util.util import to_str
+from .util.wait import wait_for_read
+
+try: # Compiled with SSL?
+ import ssl
+
+ BaseSSLError = ssl.SSLError
+except (ImportError, AttributeError):
+ ssl = None # type: ignore[assignment]
+
+ class BaseSSLError(BaseException): # type: ignore[no-redef]
+ pass
+
+
+from ._base_connection import _TYPE_BODY
+from ._base_connection import ProxyConfig as ProxyConfig
+from ._base_connection import _ResponseOptions as _ResponseOptions
+from ._version import __version__
+from .exceptions import (
+ ConnectTimeoutError,
+ HeaderParsingError,
+ NameResolutionError,
+ NewConnectionError,
+ ProxyError,
+ SystemTimeWarning,
+)
+from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
+from .util.request import body_to_chunks
+from .util.ssl_ import assert_fingerprint as _assert_fingerprint
+from .util.ssl_ import (
+ create_urllib3_context,
+ is_ipaddress,
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ ssl_wrap_socket,
+)
+from .util.ssl_match_hostname import CertificateError, match_hostname
+from .util.url import Url
+
+# Not a no-op, we're adding this to the namespace so it can be imported.
+ConnectionError = ConnectionError
+BrokenPipeError = BrokenPipeError
+
+
+log = logging.getLogger(__name__)
+
+port_by_scheme = {"http": 80, "https": 443}
+
+# When it comes time to update this value as a part of regular maintenance
+# (ie test_recent_date is failing) update it to ~6 months before the current date.
+RECENT_DATE = datetime.date(2025, 1, 1)
+
+_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
+
+
+class HTTPConnection(_HTTPConnection):
+ """
+ Based on :class:`http.client.HTTPConnection` but provides an extra constructor
+ backwards-compatibility layer between older and newer Pythons.
+
+ Additional keyword parameters are used to configure attributes of the connection.
+ Accepted parameters include:
+
+ - ``source_address``: Set the source address for the current connection.
+ - ``socket_options``: Set specific options on the underlying socket. If not specified, then
+ defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
+ Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
+
+ For example, if you wish to enable TCP Keep Alive in addition to the defaults,
+ you might pass:
+
+ .. code-block:: python
+
+ HTTPConnection.default_socket_options + [
+ (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
+ ]
+
+ Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
+ """
+
+ default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
+
+ #: Disable Nagle's algorithm by default.
+ #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
+ default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
+ (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ ]
+
+ #: Whether this connection verifies the host's certificate.
+ is_verified: bool = False
+
+ #: Whether this proxy connection verified the proxy host's certificate.
+ # If no proxy is currently connected to the value will be ``None``.
+ proxy_is_verified: bool | None = None
+
+ blocksize: int
+ source_address: tuple[str, int] | None
+ socket_options: connection._TYPE_SOCKET_OPTIONS | None
+
+ _has_connected_to_proxy: bool
+ _response_options: _ResponseOptions | None
+ _tunnel_host: str | None
+ _tunnel_port: int | None
+ _tunnel_scheme: str | None
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 16384,
+ socket_options: None | (
+ connection._TYPE_SOCKET_OPTIONS
+ ) = default_socket_options,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ ) -> None:
+ super().__init__(
+ host=host,
+ port=port,
+ timeout=Timeout.resolve_default_timeout(timeout),
+ source_address=source_address,
+ blocksize=blocksize,
+ )
+ self.socket_options = socket_options
+ self.proxy = proxy
+ self.proxy_config = proxy_config
+
+ self._has_connected_to_proxy = False
+ self._response_options = None
+ self._tunnel_host: str | None = None
+ self._tunnel_port: int | None = None
+ self._tunnel_scheme: str | None = None
+
+ @property
+ def host(self) -> str:
+ """
+ Getter method to remove any trailing dots that indicate the hostname is an FQDN.
+
+ In general, SSL certificates don't include the trailing dot indicating a
+ fully-qualified domain name, and thus, they don't validate properly when
+ checked against a domain name that includes the dot. In addition, some
+ servers may not expect to receive the trailing dot when provided.
+
+ However, the hostname with trailing dot is critical to DNS resolution; doing a
+ lookup with the trailing dot will properly only resolve the appropriate FQDN,
+ whereas a lookup without a trailing dot will search the system's search domain
+ list. Thus, it's important to keep the original host around for use only in
+ those cases where it's appropriate (i.e., when doing DNS lookup to establish the
+ actual TCP connection across which we're going to send HTTP requests).
+ """
+ return self._dns_host.rstrip(".")
+
+ @host.setter
+ def host(self, value: str) -> None:
+ """
+ Setter for the `host` property.
+
+ We assume that only urllib3 uses the _dns_host attribute; httplib itself
+ only uses `host`, and it seems reasonable that other libraries follow suit.
+ """
+ self._dns_host = value
+
+ def _new_conn(self) -> socket.socket:
+ """Establish a socket connection and set nodelay settings on it.
+
+ :return: New socket connection.
+ """
+ try:
+ sock = connection.create_connection(
+ (self._dns_host, self.port),
+ self.timeout,
+ source_address=self.source_address,
+ socket_options=self.socket_options,
+ )
+ except socket.gaierror as e:
+ raise NameResolutionError(self.host, self, e) from e
+ except SocketTimeout as e:
+ raise ConnectTimeoutError(
+ self,
+ f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
+ ) from e
+
+ except OSError as e:
+ raise NewConnectionError(
+ self, f"Failed to establish a new connection: {e}"
+ ) from e
+
+ sys.audit("http.client.connect", self, self.host, self.port)
+
+ return sock
+
+ def set_tunnel(
+ self,
+ host: str,
+ port: int | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ scheme: str = "http",
+ ) -> None:
+ if scheme not in ("http", "https"):
+ raise ValueError(
+ f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
+ )
+ super().set_tunnel(host, port=port, headers=headers)
+ self._tunnel_scheme = scheme
+
+ if sys.version_info < (3, 11, 9) or ((3, 12) <= sys.version_info < (3, 12, 3)):
+ # Taken from python/cpython#100986 which was backported in 3.11.9 and 3.12.3.
+ # When using connection_from_host, host will come without brackets.
+ def _wrap_ipv6(self, ip: bytes) -> bytes:
+ if b":" in ip and ip[0] != b"["[0]:
+ return b"[" + ip + b"]"
+ return ip
+
+ if sys.version_info < (3, 11, 9):
+ # `_tunnel` copied from 3.11.13 backporting
+ # https://github.com/python/cpython/commit/0d4026432591d43185568dd31cef6a034c4b9261
+ # and https://github.com/python/cpython/commit/6fbc61070fda2ffb8889e77e3b24bca4249ab4d1
+ def _tunnel(self) -> None:
+ _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
+ connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
+ self._wrap_ipv6(self._tunnel_host.encode("ascii")), # type: ignore[union-attr]
+ self._tunnel_port,
+ )
+ headers = [connect]
+ for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
+ headers.append(f"{header}: {value}\r\n".encode("latin-1"))
+ headers.append(b"\r\n")
+ # Making a single send() call instead of one per line encourages
+ # the host OS to use a more optimal packet size instead of
+ # potentially emitting a series of small packets.
+ self.send(b"".join(headers))
+ del headers
+
+ response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
+ try:
+ (version, code, message) = response._read_status() # type: ignore[attr-defined]
+
+ if code != http.HTTPStatus.OK:
+ self.close()
+ raise OSError(
+ f"Tunnel connection failed: {code} {message.strip()}"
+ )
+ while True:
+ line = response.fp.readline(_MAXLINE + 1)
+ if len(line) > _MAXLINE:
+ raise http.client.LineTooLong("header line")
+ if not line:
+ # for sites which EOF without sending a trailer
+ break
+ if line in (b"\r\n", b"\n", b""):
+ break
+
+ if self.debuglevel > 0:
+ print("header:", line.decode())
+ finally:
+ response.close()
+
+ elif (3, 12) <= sys.version_info < (3, 12, 3):
+ # `_tunnel` copied from 3.12.11 backporting
+ # https://github.com/python/cpython/commit/23aef575c7629abcd4aaf028ebd226fb41a4b3c8
+ def _tunnel(self) -> None: # noqa: F811
+ connect = b"CONNECT %s:%d HTTP/1.1\r\n" % ( # type: ignore[str-format]
+ self._wrap_ipv6(self._tunnel_host.encode("idna")), # type: ignore[union-attr]
+ self._tunnel_port,
+ )
+ headers = [connect]
+ for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
+ headers.append(f"{header}: {value}\r\n".encode("latin-1"))
+ headers.append(b"\r\n")
+ # Making a single send() call instead of one per line encourages
+ # the host OS to use a more optimal packet size instead of
+ # potentially emitting a series of small packets.
+ self.send(b"".join(headers))
+ del headers
+
+ response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
+ try:
+ (version, code, message) = response._read_status() # type: ignore[attr-defined]
+
+ self._raw_proxy_headers = http.client._read_headers(response.fp) # type: ignore[attr-defined]
+
+ if self.debuglevel > 0:
+ for header in self._raw_proxy_headers:
+ print("header:", header.decode())
+
+ if code != http.HTTPStatus.OK:
+ self.close()
+ raise OSError(
+ f"Tunnel connection failed: {code} {message.strip()}"
+ )
+
+ finally:
+ response.close()
+
+ def connect(self) -> None:
+ self.sock = self._new_conn()
+ if self._tunnel_host:
+ # If we're tunneling it means we're connected to our proxy.
+ self._has_connected_to_proxy = True
+
+ # TODO: Fix tunnel so it doesn't depend on self.sock state.
+ self._tunnel()
+
+ # If there's a proxy to be connected to we are fully connected.
+ # This is set twice (once above and here) due to forwarding proxies
+ # not using tunnelling.
+ self._has_connected_to_proxy = bool(self.proxy)
+
+ if self._has_connected_to_proxy:
+ self.proxy_is_verified = False
+
+ @property
+ def is_closed(self) -> bool:
+ return self.sock is None
+
+ @property
+ def is_connected(self) -> bool:
+ if self.sock is None:
+ return False
+ return not wait_for_read(self.sock, timeout=0.0)
+
+ @property
+ def has_connected_to_proxy(self) -> bool:
+ return self._has_connected_to_proxy
+
+ @property
+ def proxy_is_forwarding(self) -> bool:
+ """
+ Return True if a forwarding proxy is configured, else return False
+ """
+ return bool(self.proxy) and self._tunnel_host is None
+
+ @property
+ def proxy_is_tunneling(self) -> bool:
+ """
+ Return True if a tunneling proxy is configured, else return False
+ """
+ return self._tunnel_host is not None
+
+ def close(self) -> None:
+ try:
+ super().close()
+ finally:
+ # Reset all stateful properties so connection
+ # can be re-used without leaking prior configs.
+ self.sock = None
+ self.is_verified = False
+ self.proxy_is_verified = None
+ self._has_connected_to_proxy = False
+ self._response_options = None
+ self._tunnel_host = None
+ self._tunnel_port = None
+ self._tunnel_scheme = None
+
+ def putrequest(
+ self,
+ method: str,
+ url: str,
+ skip_host: bool = False,
+ skip_accept_encoding: bool = False,
+ ) -> None:
+ """"""
+ # Empty docstring because the indentation of CPython's implementation
+ # is broken but we don't want this method in our documentation.
+ match = _CONTAINS_CONTROL_CHAR_RE.search(method)
+ if match:
+ raise ValueError(
+ f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
+ )
+
+ return super().putrequest(
+ method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
+ )
+
+ def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
+ """"""
+ if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
+ super().putheader(header, *values)
+ elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
+ skippable_headers = "', '".join(
+ [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
+ )
+ raise ValueError(
+ f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
+ )
+
+ # `request` method's signature intentionally violates LSP.
+ # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
+ def request( # type: ignore[override]
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ *,
+ chunked: bool = False,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ enforce_content_length: bool = True,
+ ) -> None:
+ # Update the inner socket's timeout value to send the request.
+ # This only triggers if the connection is re-used.
+ if self.sock is not None:
+ self.sock.settimeout(self.timeout)
+
+ # Store these values to be fed into the HTTPResponse
+ # object later. TODO: Remove this in favor of a real
+ # HTTP lifecycle mechanism.
+
+ # We have to store these before we call .request()
+ # because sometimes we can still salvage a response
+ # off the wire even if we aren't able to completely
+ # send the request body.
+ self._response_options = _ResponseOptions(
+ request_method=method,
+ request_url=url,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ enforce_content_length=enforce_content_length,
+ )
+
+ if headers is None:
+ headers = {}
+ header_keys = frozenset(to_str(k.lower()) for k in headers)
+ skip_accept_encoding = "accept-encoding" in header_keys
+ skip_host = "host" in header_keys
+ self.putrequest(
+ method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
+ )
+
+ # Transform the body into an iterable of sendall()-able chunks
+ # and detect if an explicit Content-Length is doable.
+ chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
+ chunks = chunks_and_cl.chunks
+ content_length = chunks_and_cl.content_length
+
+ # When chunked is explicit set to 'True' we respect that.
+ if chunked:
+ if "transfer-encoding" not in header_keys:
+ self.putheader("Transfer-Encoding", "chunked")
+ else:
+ # Detect whether a framing mechanism is already in use. If so
+ # we respect that value, otherwise we pick chunked vs content-length
+ # depending on the type of 'body'.
+ if "content-length" in header_keys:
+ chunked = False
+ elif "transfer-encoding" in header_keys:
+ chunked = True
+
+ # Otherwise we go off the recommendation of 'body_to_chunks()'.
+ else:
+ chunked = False
+ if content_length is None:
+ if chunks is not None:
+ chunked = True
+ self.putheader("Transfer-Encoding", "chunked")
+ else:
+ self.putheader("Content-Length", str(content_length))
+
+ # Now that framing headers are out of the way we send all the other headers.
+ if "user-agent" not in header_keys:
+ self.putheader("User-Agent", _get_default_user_agent())
+ for header, value in headers.items():
+ self.putheader(header, value)
+ self.endheaders()
+
+ # If we're given a body we start sending that in chunks.
+ if chunks is not None:
+ for chunk in chunks:
+ # Sending empty chunks isn't allowed for TE: chunked
+ # as it indicates the end of the body.
+ if not chunk:
+ continue
+ if isinstance(chunk, str):
+ chunk = chunk.encode("utf-8")
+ if chunked:
+ self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
+ else:
+ self.send(chunk)
+
+ # Regardless of whether we have a body or not, if we're in
+ # chunked mode we want to send an explicit empty chunk.
+ if chunked:
+ self.send(b"0\r\n\r\n")
+
+ def request_chunked(
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ ) -> None:
+ """
+ Alternative to the common request method, which sends the
+ body with chunked encoding and not as one block
+ """
+ warnings.warn(
+ "HTTPConnection.request_chunked() is deprecated and will be removed "
+ "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+ self.request(method, url, body=body, headers=headers, chunked=True)
+
+ def getresponse( # type: ignore[override]
+ self,
+ ) -> HTTPResponse:
+ """
+ Get the response from the server.
+
+ If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
+
+ If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
+ """
+ # Raise the same error as http.client.HTTPConnection
+ if self._response_options is None:
+ raise ResponseNotReady()
+
+ # Reset this attribute for being used again.
+ resp_options = self._response_options
+ self._response_options = None
+
+ # Since the connection's timeout value may have been updated
+ # we need to set the timeout on the socket.
+ self.sock.settimeout(self.timeout)
+
+ # This is needed here to avoid circular import errors
+ from .response import HTTPResponse
+
+ # Save a reference to the shutdown function before ownership is passed
+ # to httplib_response
+ # TODO should we implement it everywhere?
+ _shutdown = getattr(self.sock, "shutdown", None)
+
+ # Get the response from http.client.HTTPConnection
+ httplib_response = super().getresponse()
+
+ try:
+ assert_header_parsing(httplib_response.msg)
+ except (HeaderParsingError, TypeError) as hpe:
+ log.warning(
+ "Failed to parse headers (url=%s): %s",
+ _url_from_connection(self, resp_options.request_url),
+ hpe,
+ exc_info=True,
+ )
+
+ headers = HTTPHeaderDict(httplib_response.msg.items())
+
+ response = HTTPResponse(
+ body=httplib_response,
+ headers=headers,
+ status=httplib_response.status,
+ version=httplib_response.version,
+ version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
+ reason=httplib_response.reason,
+ preload_content=resp_options.preload_content,
+ decode_content=resp_options.decode_content,
+ original_response=httplib_response,
+ enforce_content_length=resp_options.enforce_content_length,
+ request_method=resp_options.request_method,
+ request_url=resp_options.request_url,
+ sock_shutdown=_shutdown,
+ )
+ return response
+
+
+class HTTPSConnection(HTTPConnection):
+ """
+ Many of the parameters to this constructor are passed to the underlying SSL
+ socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
+ """
+
+ default_port = port_by_scheme["https"] # type: ignore[misc]
+
+ cert_reqs: int | str | None = None
+ ca_certs: str | None = None
+ ca_cert_dir: str | None = None
+ ca_cert_data: None | str | bytes = None
+ ssl_version: int | str | None = None
+ ssl_minimum_version: int | None = None
+ ssl_maximum_version: int | None = None
+ assert_fingerprint: str | None = None
+ _connect_callback: typing.Callable[..., None] | None = None
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 16384,
+ socket_options: None | (
+ connection._TYPE_SOCKET_OPTIONS
+ ) = HTTPConnection.default_socket_options,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ cert_reqs: int | str | None = None,
+ assert_hostname: None | str | typing.Literal[False] = None,
+ assert_fingerprint: str | None = None,
+ server_hostname: str | None = None,
+ ssl_context: ssl.SSLContext | None = None,
+ ca_certs: str | None = None,
+ ca_cert_dir: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ ssl_minimum_version: int | None = None,
+ ssl_maximum_version: int | None = None,
+ ssl_version: int | str | None = None, # Deprecated
+ cert_file: str | None = None,
+ key_file: str | None = None,
+ key_password: str | None = None,
+ ) -> None:
+ super().__init__(
+ host,
+ port=port,
+ timeout=timeout,
+ source_address=source_address,
+ blocksize=blocksize,
+ socket_options=socket_options,
+ proxy=proxy,
+ proxy_config=proxy_config,
+ )
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.key_password = key_password
+ self.ssl_context = ssl_context
+ self.server_hostname = server_hostname
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+ self.ssl_version = ssl_version
+ self.ssl_minimum_version = ssl_minimum_version
+ self.ssl_maximum_version = ssl_maximum_version
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
+ self.ca_cert_data = ca_cert_data
+
+ # cert_reqs depends on ssl_context so calculate last.
+ if cert_reqs is None:
+ if self.ssl_context is not None:
+ cert_reqs = self.ssl_context.verify_mode
+ else:
+ cert_reqs = resolve_cert_reqs(None)
+ self.cert_reqs = cert_reqs
+ self._connect_callback = None
+
+ def set_cert(
+ self,
+ key_file: str | None = None,
+ cert_file: str | None = None,
+ cert_reqs: int | str | None = None,
+ key_password: str | None = None,
+ ca_certs: str | None = None,
+ assert_hostname: None | str | typing.Literal[False] = None,
+ assert_fingerprint: str | None = None,
+ ca_cert_dir: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ ) -> None:
+ """
+ This method should only be called once, before the connection is used.
+ """
+ warnings.warn(
+ "HTTPSConnection.set_cert() is deprecated and will be removed "
+ "in urllib3 v2.1.0. Instead provide the parameters to the "
+ "HTTPSConnection constructor.",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+
+ # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
+ # have an SSLContext object in which case we'll use its verify_mode.
+ if cert_reqs is None:
+ if self.ssl_context is not None:
+ cert_reqs = self.ssl_context.verify_mode
+ else:
+ cert_reqs = resolve_cert_reqs(None)
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.cert_reqs = cert_reqs
+ self.key_password = key_password
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
+ self.ca_cert_data = ca_cert_data
+
+ def connect(self) -> None:
+ # Today we don't need to be doing this step before the /actual/ socket
+ # connection, however in the future we'll need to decide whether to
+ # create a new socket or re-use an existing "shared" socket as a part
+ # of the HTTP/2 handshake dance.
+ if self._tunnel_host is not None and self._tunnel_port is not None:
+ probe_http2_host = self._tunnel_host
+ probe_http2_port = self._tunnel_port
+ else:
+ probe_http2_host = self.host
+ probe_http2_port = self.port
+
+ # Check if the target origin supports HTTP/2.
+ # If the value comes back as 'None' it means that the current thread
+ # is probing for HTTP/2 support. Otherwise, we're waiting for another
+ # probe to complete, or we get a value right away.
+ target_supports_http2: bool | None
+ if "h2" in ssl_.ALPN_PROTOCOLS:
+ target_supports_http2 = http2_probe.acquire_and_get(
+ host=probe_http2_host, port=probe_http2_port
+ )
+ else:
+ # If HTTP/2 isn't going to be offered it doesn't matter if
+ # the target supports HTTP/2. Don't want to make a probe.
+ target_supports_http2 = False
+
+ if self._connect_callback is not None:
+ self._connect_callback(
+ "before connect",
+ thread_id=threading.get_ident(),
+ target_supports_http2=target_supports_http2,
+ )
+
+ try:
+ sock: socket.socket | ssl.SSLSocket
+ self.sock = sock = self._new_conn()
+ server_hostname: str = self.host
+ tls_in_tls = False
+
+ # Do we need to establish a tunnel?
+ if self.proxy_is_tunneling:
+ # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
+ if self._tunnel_scheme == "https":
+ # _connect_tls_proxy will verify and assign proxy_is_verified
+ self.sock = sock = self._connect_tls_proxy(self.host, sock)
+ tls_in_tls = True
+ elif self._tunnel_scheme == "http":
+ self.proxy_is_verified = False
+
+ # If we're tunneling it means we're connected to our proxy.
+ self._has_connected_to_proxy = True
+
+ self._tunnel()
+ # Override the host with the one we're requesting data from.
+ server_hostname = typing.cast(str, self._tunnel_host)
+
+ if self.server_hostname is not None:
+ server_hostname = self.server_hostname
+
+ is_time_off = datetime.date.today() < RECENT_DATE
+ if is_time_off:
+ warnings.warn(
+ (
+ f"System time is way off (before {RECENT_DATE}). This will probably "
+ "lead to SSL verification errors"
+ ),
+ SystemTimeWarning,
+ )
+
+ # Remove trailing '.' from fqdn hostnames to allow certificate validation
+ server_hostname_rm_dot = server_hostname.rstrip(".")
+
+ sock_and_verified = _ssl_wrap_socket_and_match_hostname(
+ sock=sock,
+ cert_reqs=self.cert_reqs,
+ ssl_version=self.ssl_version,
+ ssl_minimum_version=self.ssl_minimum_version,
+ ssl_maximum_version=self.ssl_maximum_version,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ ca_cert_data=self.ca_cert_data,
+ cert_file=self.cert_file,
+ key_file=self.key_file,
+ key_password=self.key_password,
+ server_hostname=server_hostname_rm_dot,
+ ssl_context=self.ssl_context,
+ tls_in_tls=tls_in_tls,
+ assert_hostname=self.assert_hostname,
+ assert_fingerprint=self.assert_fingerprint,
+ )
+ self.sock = sock_and_verified.socket
+
+ # If an error occurs during connection/handshake we may need to release
+ # our lock so another connection can probe the origin.
+ except BaseException:
+ if self._connect_callback is not None:
+ self._connect_callback(
+ "after connect failure",
+ thread_id=threading.get_ident(),
+ target_supports_http2=target_supports_http2,
+ )
+
+ if target_supports_http2 is None:
+ http2_probe.set_and_release(
+ host=probe_http2_host, port=probe_http2_port, supports_http2=None
+ )
+ raise
+
+ # If this connection doesn't know if the origin supports HTTP/2
+ # we report back to the HTTP/2 probe our result.
+ if target_supports_http2 is None:
+ supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2"
+ http2_probe.set_and_release(
+ host=probe_http2_host,
+ port=probe_http2_port,
+ supports_http2=supports_http2,
+ )
+
+ # Forwarding proxies can never have a verified target since
+ # the proxy is the one doing the verification. Should instead
+ # use a CONNECT tunnel in order to verify the target.
+ # See: https://github.com/urllib3/urllib3/issues/3267.
+ if self.proxy_is_forwarding:
+ self.is_verified = False
+ else:
+ self.is_verified = sock_and_verified.is_verified
+
+ # If there's a proxy to be connected to we are fully connected.
+ # This is set twice (once above and here) due to forwarding proxies
+ # not using tunnelling.
+ self._has_connected_to_proxy = bool(self.proxy)
+
+ # Set `self.proxy_is_verified` unless it's already set while
+ # establishing a tunnel.
+ if self._has_connected_to_proxy and self.proxy_is_verified is None:
+ self.proxy_is_verified = sock_and_verified.is_verified
+
+ def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
+ """
+ Establish a TLS connection to the proxy using the provided SSL context.
+ """
+ # `_connect_tls_proxy` is called when self._tunnel_host is truthy.
+ proxy_config = typing.cast(ProxyConfig, self.proxy_config)
+ ssl_context = proxy_config.ssl_context
+ sock_and_verified = _ssl_wrap_socket_and_match_hostname(
+ sock,
+ cert_reqs=self.cert_reqs,
+ ssl_version=self.ssl_version,
+ ssl_minimum_version=self.ssl_minimum_version,
+ ssl_maximum_version=self.ssl_maximum_version,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ ca_cert_data=self.ca_cert_data,
+ server_hostname=hostname,
+ ssl_context=ssl_context,
+ assert_hostname=proxy_config.assert_hostname,
+ assert_fingerprint=proxy_config.assert_fingerprint,
+ # Features that aren't implemented for proxies yet:
+ cert_file=None,
+ key_file=None,
+ key_password=None,
+ tls_in_tls=False,
+ )
+ self.proxy_is_verified = sock_and_verified.is_verified
+ return sock_and_verified.socket # type: ignore[return-value]
+
+
+class _WrappedAndVerifiedSocket(typing.NamedTuple):
+ """
+ Wrapped socket and whether the connection is
+ verified after the TLS handshake
+ """
+
+ socket: ssl.SSLSocket | SSLTransport
+ is_verified: bool
+
+
+def _ssl_wrap_socket_and_match_hostname(
+ sock: socket.socket,
+ *,
+ cert_reqs: None | str | int,
+ ssl_version: None | str | int,
+ ssl_minimum_version: int | None,
+ ssl_maximum_version: int | None,
+ cert_file: str | None,
+ key_file: str | None,
+ key_password: str | None,
+ ca_certs: str | None,
+ ca_cert_dir: str | None,
+ ca_cert_data: None | str | bytes,
+ assert_hostname: None | str | typing.Literal[False],
+ assert_fingerprint: str | None,
+ server_hostname: str | None,
+ ssl_context: ssl.SSLContext | None,
+ tls_in_tls: bool = False,
+) -> _WrappedAndVerifiedSocket:
+ """Logic for constructing an SSLContext from all TLS parameters, passing
+ that down into ssl_wrap_socket, and then doing certificate verification
+ either via hostname or fingerprint. This function exists to guarantee
+ that both proxies and targets have the same behavior when connecting via TLS.
+ """
+ default_ssl_context = False
+ if ssl_context is None:
+ default_ssl_context = True
+ context = create_urllib3_context(
+ ssl_version=resolve_ssl_version(ssl_version),
+ ssl_minimum_version=ssl_minimum_version,
+ ssl_maximum_version=ssl_maximum_version,
+ cert_reqs=resolve_cert_reqs(cert_reqs),
+ )
+ else:
+ context = ssl_context
+
+ context.verify_mode = resolve_cert_reqs(cert_reqs)
+
+ # In some cases, we want to verify hostnames ourselves
+ if (
+ # `ssl` can't verify fingerprints or alternate hostnames
+ assert_fingerprint
+ or assert_hostname
+ # assert_hostname can be set to False to disable hostname checking
+ or assert_hostname is False
+ # We still support OpenSSL 1.0.2, which prevents us from verifying
+ # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
+ or ssl_.IS_PYOPENSSL
+ or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
+ ):
+ context.check_hostname = False
+
+ # Try to load OS default certs if none are given. We need to do the hasattr() check
+ # for custom pyOpenSSL SSLContext objects because they don't support
+ # load_default_certs().
+ if (
+ not ca_certs
+ and not ca_cert_dir
+ and not ca_cert_data
+ and default_ssl_context
+ and hasattr(context, "load_default_certs")
+ ):
+ context.load_default_certs()
+
+ # Ensure that IPv6 addresses are in the proper format and don't have a
+ # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
+ # and interprets them as DNS hostnames.
+ if server_hostname is not None:
+ normalized = server_hostname.strip("[]")
+ if "%" in normalized:
+ normalized = normalized[: normalized.rfind("%")]
+ if is_ipaddress(normalized):
+ server_hostname = normalized
+
+ ssl_sock = ssl_wrap_socket(
+ sock=sock,
+ keyfile=key_file,
+ certfile=cert_file,
+ key_password=key_password,
+ ca_certs=ca_certs,
+ ca_cert_dir=ca_cert_dir,
+ ca_cert_data=ca_cert_data,
+ server_hostname=server_hostname,
+ ssl_context=context,
+ tls_in_tls=tls_in_tls,
+ )
+
+ try:
+ if assert_fingerprint:
+ _assert_fingerprint(
+ ssl_sock.getpeercert(binary_form=True), assert_fingerprint
+ )
+ elif (
+ context.verify_mode != ssl.CERT_NONE
+ and not context.check_hostname
+ and assert_hostname is not False
+ ):
+ cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
+
+ # Need to signal to our match_hostname whether to use 'commonName' or not.
+ # If we're using our own constructed SSLContext we explicitly set 'False'
+ # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
+ if default_ssl_context:
+ hostname_checks_common_name = False
+ else:
+ hostname_checks_common_name = (
+ getattr(context, "hostname_checks_common_name", False) or False
+ )
+
+ _match_hostname(
+ cert,
+ assert_hostname or server_hostname, # type: ignore[arg-type]
+ hostname_checks_common_name,
+ )
+
+ return _WrappedAndVerifiedSocket(
+ socket=ssl_sock,
+ is_verified=context.verify_mode == ssl.CERT_REQUIRED
+ or bool(assert_fingerprint),
+ )
+ except BaseException:
+ ssl_sock.close()
+ raise
+
+
+def _match_hostname(
+ cert: _TYPE_PEER_CERT_RET_DICT | None,
+ asserted_hostname: str,
+ hostname_checks_common_name: bool = False,
+) -> None:
+ # Our upstream implementation of ssl.match_hostname()
+ # only applies this normalization to IP addresses so it doesn't
+ # match DNS SANs so we do the same thing!
+ stripped_hostname = asserted_hostname.strip("[]")
+ if is_ipaddress(stripped_hostname):
+ asserted_hostname = stripped_hostname
+
+ try:
+ match_hostname(cert, asserted_hostname, hostname_checks_common_name)
+ except CertificateError as e:
+ log.warning(
+ "Certificate did not match expected hostname: %s. Certificate: %s",
+ asserted_hostname,
+ cert,
+ )
+ # Add cert to exception and reraise so client code can inspect
+ # the cert when catching the exception, if they want to
+ e._peer_cert = cert # type: ignore[attr-defined]
+ raise
+
+
+def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
+ # Look for the phrase 'wrong version number', if found
+ # then we should warn the user that we're very sure that
+ # this proxy is HTTP-only and they have a configuration issue.
+ error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
+ is_likely_http_proxy = (
+ "wrong version number" in error_normalized
+ or "unknown protocol" in error_normalized
+ or "record layer failure" in error_normalized
+ )
+ http_proxy_warning = (
+ ". Your proxy appears to only use HTTP and not HTTPS, "
+ "try changing your proxy URL to be HTTP. See: "
+ "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
+ "#https-proxy-error-http-proxy"
+ )
+ new_err = ProxyError(
+ f"Unable to connect to proxy"
+ f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
+ err,
+ )
+ new_err.__cause__ = err
+ return new_err
+
+
+def _get_default_user_agent() -> str:
+ return f"python-urllib3/{__version__}"
+
+
+class DummyConnection:
+ """Used to detect a failed ConnectionCls import."""
+
+
+if not ssl:
+ HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
+
+
+VerifiedHTTPSConnection = HTTPSConnection
+
+
+def _url_from_connection(
+ conn: HTTPConnection | HTTPSConnection, path: str | None = None
+) -> str:
+ """Returns the URL from a given connection. This is mainly used for testing and logging."""
+
+ scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
+
+ return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url
diff --git a/phivenv/Lib/site-packages/urllib3/connectionpool.py b/phivenv/Lib/site-packages/urllib3/connectionpool.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a0685b4cdd0562e508b9dd032765b5c759ea61e
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/connectionpool.py
@@ -0,0 +1,1178 @@
+from __future__ import annotations
+
+import errno
+import logging
+import queue
+import sys
+import typing
+import warnings
+import weakref
+from socket import timeout as SocketTimeout
+from types import TracebackType
+
+from ._base_connection import _TYPE_BODY
+from ._collections import HTTPHeaderDict
+from ._request_methods import RequestMethods
+from .connection import (
+ BaseSSLError,
+ BrokenPipeError,
+ DummyConnection,
+ HTTPConnection,
+ HTTPException,
+ HTTPSConnection,
+ ProxyConfig,
+ _wrap_proxy_error,
+)
+from .connection import port_by_scheme as port_by_scheme
+from .exceptions import (
+ ClosedPoolError,
+ EmptyPoolError,
+ FullPoolError,
+ HostChangedError,
+ InsecureRequestWarning,
+ LocationValueError,
+ MaxRetryError,
+ NewConnectionError,
+ ProtocolError,
+ ProxyError,
+ ReadTimeoutError,
+ SSLError,
+ TimeoutError,
+)
+from .response import BaseHTTPResponse
+from .util.connection import is_connection_dropped
+from .util.proxy import connection_requires_http_tunnel
+from .util.request import _TYPE_BODY_POSITION, set_file_position
+from .util.retry import Retry
+from .util.ssl_match_hostname import CertificateError
+from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
+from .util.url import Url, _encode_target
+from .util.url import _normalize_host as normalize_host
+from .util.url import parse_url
+from .util.util import to_str
+
+if typing.TYPE_CHECKING:
+ import ssl
+
+ from typing_extensions import Self
+
+ from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
+
+log = logging.getLogger(__name__)
+
+_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
+
+
+# Pool objects
+class ConnectionPool:
+ """
+ Base class for all connection pools, such as
+ :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
+
+ .. note::
+ ConnectionPool.urlopen() does not normalize or percent-encode target URIs
+ which is useful if your target server doesn't support percent-encoded
+ target URIs.
+ """
+
+ scheme: str | None = None
+ QueueCls = queue.LifoQueue
+
+ def __init__(self, host: str, port: int | None = None) -> None:
+ if not host:
+ raise LocationValueError("No host specified.")
+
+ self.host = _normalize_host(host, scheme=self.scheme)
+ self.port = port
+
+ # This property uses 'normalize_host()' (not '_normalize_host()')
+ # to avoid removing square braces around IPv6 addresses.
+ # This value is sent to `HTTPConnection.set_tunnel()` if called
+ # because square braces are required for HTTP CONNECT tunneling.
+ self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()
+
+ def __str__(self) -> str:
+ return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> typing.Literal[False]:
+ self.close()
+ # Return False to re-raise any potential exceptions
+ return False
+
+ def close(self) -> None:
+ """
+ Close all pooled connections and disable the pool.
+ """
+
+
+# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
+_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
+
+
+class HTTPConnectionPool(ConnectionPool, RequestMethods):
+ """
+ Thread-safe connection pool for one host.
+
+ :param host:
+ Host used for this HTTP Connection (e.g. "localhost"), passed into
+ :class:`http.client.HTTPConnection`.
+
+ :param port:
+ Port used for this HTTP Connection (None is equivalent to 80), passed
+ into :class:`http.client.HTTPConnection`.
+
+ :param timeout:
+ Socket timeout in seconds for each individual connection. This can
+ be a float or integer, which sets the timeout for the HTTP request,
+ or an instance of :class:`urllib3.util.Timeout` which gives you more
+ fine-grained control over request timeouts. After the constructor has
+ been parsed, this is always a `urllib3.util.Timeout` object.
+
+ :param maxsize:
+ Number of connections to save that can be reused. More than 1 is useful
+ in multithreaded situations. If ``block`` is set to False, more
+ connections will be created but they will not be saved once they've
+ been used.
+
+ :param block:
+ If set to True, no more than ``maxsize`` connections will be used at
+ a time. When no free connections are available, the call will block
+ until a connection has been released. This is a useful side effect for
+ particular multithreaded situations where one does not want to use more
+ than maxsize connections per host to prevent flooding.
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+
+ :param retries:
+ Retry configuration to use by default with requests in this pool.
+
+ :param _proxy:
+ Parsed proxy URL, should not be used directly, instead, see
+ :class:`urllib3.ProxyManager`
+
+ :param _proxy_headers:
+ A dictionary with proxy headers, should not be used directly,
+ instead, see :class:`urllib3.ProxyManager`
+
+ :param \\**conn_kw:
+ Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
+ :class:`urllib3.connection.HTTPSConnection` instances.
+ """
+
+ scheme = "http"
+ ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
+ maxsize: int = 1,
+ block: bool = False,
+ headers: typing.Mapping[str, str] | None = None,
+ retries: Retry | bool | int | None = None,
+ _proxy: Url | None = None,
+ _proxy_headers: typing.Mapping[str, str] | None = None,
+ _proxy_config: ProxyConfig | None = None,
+ **conn_kw: typing.Any,
+ ):
+ ConnectionPool.__init__(self, host, port)
+ RequestMethods.__init__(self, headers)
+
+ if not isinstance(timeout, Timeout):
+ timeout = Timeout.from_float(timeout)
+
+ if retries is None:
+ retries = Retry.DEFAULT
+
+ self.timeout = timeout
+ self.retries = retries
+
+ self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)
+ self.block = block
+
+ self.proxy = _proxy
+ self.proxy_headers = _proxy_headers or {}
+ self.proxy_config = _proxy_config
+
+ # Fill the queue up so that doing get() on it will block properly
+ for _ in range(maxsize):
+ self.pool.put(None)
+
+ # These are mostly for testing and debugging purposes.
+ self.num_connections = 0
+ self.num_requests = 0
+ self.conn_kw = conn_kw
+
+ if self.proxy:
+ # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
+ # We cannot know if the user has added default socket options, so we cannot replace the
+ # list.
+ self.conn_kw.setdefault("socket_options", [])
+
+ self.conn_kw["proxy"] = self.proxy
+ self.conn_kw["proxy_config"] = self.proxy_config
+
+ # Do not pass 'self' as callback to 'finalize'.
+ # Then the 'finalize' would keep an endless living (leak) to self.
+ # By just passing a reference to the pool allows the garbage collector
+ # to free self if nobody else has a reference to it.
+ pool = self.pool
+
+ # Close all the HTTPConnections in the pool before the
+ # HTTPConnectionPool object is garbage collected.
+ weakref.finalize(self, _close_pool_connections, pool)
+
+ def _new_conn(self) -> BaseHTTPConnection:
+ """
+ Return a fresh :class:`HTTPConnection`.
+ """
+ self.num_connections += 1
+ log.debug(
+ "Starting new HTTP connection (%d): %s:%s",
+ self.num_connections,
+ self.host,
+ self.port or "80",
+ )
+
+ conn = self.ConnectionCls(
+ host=self.host,
+ port=self.port,
+ timeout=self.timeout.connect_timeout,
+ **self.conn_kw,
+ )
+ return conn
+
+ def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
+ """
+ Get a connection. Will return a pooled connection if one is available.
+
+ If no connections are available and :prop:`.block` is ``False``, then a
+ fresh connection is returned.
+
+ :param timeout:
+ Seconds to wait before giving up and raising
+ :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
+ :prop:`.block` is ``True``.
+ """
+ conn = None
+
+ if self.pool is None:
+ raise ClosedPoolError(self, "Pool is closed.")
+
+ try:
+ conn = self.pool.get(block=self.block, timeout=timeout)
+
+ except AttributeError: # self.pool is None
+ raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
+
+ except queue.Empty:
+ if self.block:
+ raise EmptyPoolError(
+ self,
+ "Pool is empty and a new connection can't be opened due to blocking mode.",
+ ) from None
+ pass # Oh well, we'll create a new connection then
+
+ # If this is a persistent connection, check if it got disconnected
+ if conn and is_connection_dropped(conn):
+ log.debug("Resetting dropped connection: %s", self.host)
+ conn.close()
+
+ return conn or self._new_conn()
+
+ def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
+ """
+ Put a connection back into the pool.
+
+ :param conn:
+ Connection object for the current host and port as returned by
+ :meth:`._new_conn` or :meth:`._get_conn`.
+
+ If the pool is already full, the connection is closed and discarded
+ because we exceeded maxsize. If connections are discarded frequently,
+ then maxsize should be increased.
+
+ If the pool is closed, then the connection will be closed and discarded.
+ """
+ if self.pool is not None:
+ try:
+ self.pool.put(conn, block=False)
+ return # Everything is dandy, done.
+ except AttributeError:
+ # self.pool is None.
+ pass
+ except queue.Full:
+ # Connection never got put back into the pool, close it.
+ if conn:
+ conn.close()
+
+ if self.block:
+ # This should never happen if you got the conn from self._get_conn
+ raise FullPoolError(
+ self,
+ "Pool reached maximum size and no more connections are allowed.",
+ ) from None
+
+ log.warning(
+ "Connection pool is full, discarding connection: %s. Connection pool size: %s",
+ self.host,
+ self.pool.qsize(),
+ )
+
+ # Connection never got put back into the pool, close it.
+ if conn:
+ conn.close()
+
+ def _validate_conn(self, conn: BaseHTTPConnection) -> None:
+ """
+ Called right before a request is made, after the socket is created.
+ """
+
+ def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
+ # Nothing to do for HTTP connections.
+ pass
+
+ def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
+ """Helper that always returns a :class:`urllib3.util.Timeout`"""
+ if timeout is _DEFAULT_TIMEOUT:
+ return self.timeout.clone()
+
+ if isinstance(timeout, Timeout):
+ return timeout.clone()
+ else:
+ # User passed us an int/float. This is for backwards compatibility,
+ # can be removed later
+ return Timeout.from_float(timeout)
+
+ def _raise_timeout(
+ self,
+ err: BaseSSLError | OSError | SocketTimeout,
+ url: str,
+ timeout_value: _TYPE_TIMEOUT | None,
+ ) -> None:
+ """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
+
+ if isinstance(err, SocketTimeout):
+ raise ReadTimeoutError(
+ self, url, f"Read timed out. (read timeout={timeout_value})"
+ ) from err
+
+ # See the above comment about EAGAIN in Python 3.
+ if hasattr(err, "errno") and err.errno in _blocking_errnos:
+ raise ReadTimeoutError(
+ self, url, f"Read timed out. (read timeout={timeout_value})"
+ ) from err
+
+ def _make_request(
+ self,
+ conn: BaseHTTPConnection,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ retries: Retry | None = None,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ chunked: bool = False,
+ response_conn: BaseHTTPConnection | None = None,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ enforce_content_length: bool = True,
+ ) -> BaseHTTPResponse:
+ """
+ Perform a request on a given urllib connection object taken from our
+ pool.
+
+ :param conn:
+ a connection from one of our connection pools
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param body:
+ Data to send in the request body, either :class:`str`, :class:`bytes`,
+ an iterable of :class:`str`/:class:`bytes`, or a file-like object.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+
+ :param retries:
+ Configure the number of retries to allow before raising a
+ :class:`~urllib3.exceptions.MaxRetryError` exception.
+
+ Pass ``None`` to retry until you receive a response. Pass a
+ :class:`~urllib3.util.retry.Retry` object for fine-grained control
+ over different types of retries.
+ Pass an integer number to retry connection errors that many times,
+ but no other types of errors. Pass zero to never retry.
+
+ If ``False``, then retries are disabled and any exception is raised
+ immediately. Also, instead of raising a MaxRetryError on redirects,
+ the redirect response will be returned.
+
+ :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
+
+ :param timeout:
+ If specified, overrides the default timeout for this one
+ request. It may be a float (in seconds) or an instance of
+ :class:`urllib3.util.Timeout`.
+
+ :param chunked:
+ If True, urllib3 will send the body using chunked transfer
+ encoding. Otherwise, urllib3 will send the body using the standard
+ content-length form. Defaults to False.
+
+ :param response_conn:
+ Set this to ``None`` if you will handle releasing the connection or
+ set the connection to have the response release it.
+
+ :param preload_content:
+ If True, the response's body will be preloaded during construction.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param enforce_content_length:
+ Enforce content length checking. Body returned by server must match
+ value of Content-Length header, if present. Otherwise, raise error.
+ """
+ self.num_requests += 1
+
+ timeout_obj = self._get_timeout(timeout)
+ timeout_obj.start_connect()
+ conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
+
+ try:
+ # Trigger any extra validation we need to do.
+ try:
+ self._validate_conn(conn)
+ except (SocketTimeout, BaseSSLError) as e:
+ self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
+ raise
+
+ # _validate_conn() starts the connection to an HTTPS proxy
+ # so we need to wrap errors with 'ProxyError' here too.
+ except (
+ OSError,
+ NewConnectionError,
+ TimeoutError,
+ BaseSSLError,
+ CertificateError,
+ SSLError,
+ ) as e:
+ new_e: Exception = e
+ if isinstance(e, (BaseSSLError, CertificateError)):
+ new_e = SSLError(e)
+ # If the connection didn't successfully connect to it's proxy
+ # then there
+ if isinstance(
+ new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
+ ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
+ new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
+ raise new_e
+
+ # conn.request() calls http.client.*.request, not the method in
+ # urllib3.request. It also calls makefile (recv) on the socket.
+ try:
+ conn.request(
+ method,
+ url,
+ body=body,
+ headers=headers,
+ chunked=chunked,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ enforce_content_length=enforce_content_length,
+ )
+
+ # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
+ # legitimately able to close the connection after sending a valid response.
+ # With this behaviour, the received response is still readable.
+ except BrokenPipeError:
+ pass
+ except OSError as e:
+ # MacOS/Linux
+ # EPROTOTYPE and ECONNRESET are needed on macOS
+ # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
+ # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
+ if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
+ raise
+
+ # Reset the timeout for the recv() on the socket
+ read_timeout = timeout_obj.read_timeout
+
+ if not conn.is_closed:
+ # In Python 3 socket.py will catch EAGAIN and return None when you
+ # try and read into the file pointer created by http.client, which
+ # instead raises a BadStatusLine exception. Instead of catching
+ # the exception and assuming all BadStatusLine exceptions are read
+ # timeouts, check for a zero timeout before making the request.
+ if read_timeout == 0:
+ raise ReadTimeoutError(
+ self, url, f"Read timed out. (read timeout={read_timeout})"
+ )
+ conn.timeout = read_timeout
+
+ # Receive the response from the server
+ try:
+ response = conn.getresponse()
+ except (BaseSSLError, OSError) as e:
+ self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
+ raise
+
+ # Set properties that are used by the pooling layer.
+ response.retries = retries
+ response._connection = response_conn # type: ignore[attr-defined]
+ response._pool = self # type: ignore[attr-defined]
+
+ log.debug(
+ '%s://%s:%s "%s %s %s" %s %s',
+ self.scheme,
+ self.host,
+ self.port,
+ method,
+ url,
+ response.version_string,
+ response.status,
+ response.length_remaining,
+ )
+
+ return response
+
+ def close(self) -> None:
+ """
+ Close all pooled connections and disable the pool.
+ """
+ if self.pool is None:
+ return
+ # Disable access to the pool
+ old_pool, self.pool = self.pool, None
+
+ # Close all the HTTPConnections in the pool.
+ _close_pool_connections(old_pool)
+
+ def is_same_host(self, url: str) -> bool:
+ """
+ Check if the given ``url`` is a member of the same host as this
+ connection pool.
+ """
+ if url.startswith("/"):
+ return True
+
+ # TODO: Add optional support for socket.gethostbyname checking.
+ scheme, _, host, port, *_ = parse_url(url)
+ scheme = scheme or "http"
+ if host is not None:
+ host = _normalize_host(host, scheme=scheme)
+
+ # Use explicit default port for comparison when none is given
+ if self.port and not port:
+ port = port_by_scheme.get(scheme)
+ elif not self.port and port == port_by_scheme.get(scheme):
+ port = None
+
+ return (scheme, host, port) == (self.scheme, self.host, self.port)
+
+ def urlopen( # type: ignore[override]
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ retries: Retry | bool | int | None = None,
+ redirect: bool = True,
+ assert_same_host: bool = True,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ pool_timeout: int | None = None,
+ release_conn: bool | None = None,
+ chunked: bool = False,
+ body_pos: _TYPE_BODY_POSITION | None = None,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ **response_kw: typing.Any,
+ ) -> BaseHTTPResponse:
+ """
+ Get a connection from the pool and perform an HTTP request. This is the
+ lowest level call for making a request, so you'll need to specify all
+ the raw details.
+
+ .. note::
+
+ More commonly, it's appropriate to use a convenience method
+ such as :meth:`request`.
+
+ .. note::
+
+ `release_conn` will only behave as expected if
+ `preload_content=False` because we want to make
+ `preload_content=False` the default behaviour someday soon without
+ breaking backwards compatibility.
+
+ :param method:
+ HTTP request method (such as GET, POST, PUT, etc.)
+
+ :param url:
+ The URL to perform the request on.
+
+ :param body:
+ Data to send in the request body, either :class:`str`, :class:`bytes`,
+ an iterable of :class:`str`/:class:`bytes`, or a file-like object.
+
+ :param headers:
+ Dictionary of custom headers to send, such as User-Agent,
+ If-None-Match, etc. If None, pool headers are used. If provided,
+ these headers completely replace any pool-specific headers.
+
+ :param retries:
+ Configure the number of retries to allow before raising a
+ :class:`~urllib3.exceptions.MaxRetryError` exception.
+
+ If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
+ :class:`~urllib3.util.retry.Retry` object for fine-grained control
+ over different types of retries.
+ Pass an integer number to retry connection errors that many times,
+ but no other types of errors. Pass zero to never retry.
+
+ If ``False``, then retries are disabled and any exception is raised
+ immediately. Also, instead of raising a MaxRetryError on redirects,
+ the redirect response will be returned.
+
+ :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
+
+ :param redirect:
+ If True, automatically handle redirects (status codes 301, 302,
+ 303, 307, 308). Each redirect counts as a retry. Disabling retries
+ will disable redirect, too.
+
+ :param assert_same_host:
+ If ``True``, will make sure that the host of the pool requests is
+ consistent else will raise HostChangedError. When ``False``, you can
+ use the pool on an HTTP proxy and request foreign hosts.
+
+ :param timeout:
+ If specified, overrides the default timeout for this one
+ request. It may be a float (in seconds) or an instance of
+ :class:`urllib3.util.Timeout`.
+
+ :param pool_timeout:
+ If set and the pool is set to block=True, then this method will
+ block for ``pool_timeout`` seconds and raise EmptyPoolError if no
+ connection is available within the time period.
+
+ :param bool preload_content:
+ If True, the response's body will be preloaded into memory.
+
+ :param bool decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param release_conn:
+ If False, then the urlopen call will not release the connection
+ back into the pool once a response is received (but will release if
+ you read the entire contents of the response such as when
+ `preload_content=True`). This is useful if you're not preloading
+ the response's content immediately. You will need to call
+ ``r.release_conn()`` on the response ``r`` to return the connection
+ back into the pool. If None, it takes the value of ``preload_content``
+ which defaults to ``True``.
+
+ :param bool chunked:
+ If True, urllib3 will send the body using chunked transfer
+ encoding. Otherwise, urllib3 will send the body using the standard
+ content-length form. Defaults to False.
+
+ :param int body_pos:
+ Position to seek to in file-like body in the event of a retry or
+ redirect. Typically this won't need to be set because urllib3 will
+ auto-populate the value when needed.
+ """
+ parsed_url = parse_url(url)
+ destination_scheme = parsed_url.scheme
+
+ if headers is None:
+ headers = self.headers
+
+ if not isinstance(retries, Retry):
+ retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
+
+ if release_conn is None:
+ release_conn = preload_content
+
+ # Check host
+ if assert_same_host and not self.is_same_host(url):
+ raise HostChangedError(self, url, retries)
+
+ # Ensure that the URL we're connecting to is properly encoded
+ if url.startswith("/"):
+ url = to_str(_encode_target(url))
+ else:
+ url = to_str(parsed_url.url)
+
+ conn = None
+
+ # Track whether `conn` needs to be released before
+ # returning/raising/recursing. Update this variable if necessary, and
+ # leave `release_conn` constant throughout the function. That way, if
+ # the function recurses, the original value of `release_conn` will be
+ # passed down into the recursive call, and its value will be respected.
+ #
+ # See issue #651 [1] for details.
+ #
+ # [1]
+ release_this_conn = release_conn
+
+ http_tunnel_required = connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, destination_scheme
+ )
+
+ # Merge the proxy headers. Only done when not using HTTP CONNECT. We
+ # have to copy the headers dict so we can safely change it without those
+ # changes being reflected in anyone else's copy.
+ if not http_tunnel_required:
+ headers = headers.copy() # type: ignore[attr-defined]
+ headers.update(self.proxy_headers) # type: ignore[union-attr]
+
+ # Must keep the exception bound to a separate variable or else Python 3
+ # complains about UnboundLocalError.
+ err = None
+
+ # Keep track of whether we cleanly exited the except block. This
+ # ensures we do proper cleanup in finally.
+ clean_exit = False
+
+ # Rewind body position, if needed. Record current position
+ # for future rewinds in the event of a redirect/retry.
+ body_pos = set_file_position(body, body_pos)
+
+ try:
+ # Request a connection from the queue.
+ timeout_obj = self._get_timeout(timeout)
+ conn = self._get_conn(timeout=pool_timeout)
+
+ conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
+
+ # Is this a closed/new connection that requires CONNECT tunnelling?
+ if self.proxy is not None and http_tunnel_required and conn.is_closed:
+ try:
+ self._prepare_proxy(conn)
+ except (BaseSSLError, OSError, SocketTimeout) as e:
+ self._raise_timeout(
+ err=e, url=self.proxy.url, timeout_value=conn.timeout
+ )
+ raise
+
+ # If we're going to release the connection in ``finally:``, then
+ # the response doesn't need to know about the connection. Otherwise
+ # it will also try to release it and we'll have a double-release
+ # mess.
+ response_conn = conn if not release_conn else None
+
+ # Make the request on the HTTPConnection object
+ response = self._make_request(
+ conn,
+ method,
+ url,
+ timeout=timeout_obj,
+ body=body,
+ headers=headers,
+ chunked=chunked,
+ retries=retries,
+ response_conn=response_conn,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ **response_kw,
+ )
+
+ # Everything went great!
+ clean_exit = True
+
+ except EmptyPoolError:
+ # Didn't get a connection from the pool, no need to clean up
+ clean_exit = True
+ release_this_conn = False
+ raise
+
+ except (
+ TimeoutError,
+ HTTPException,
+ OSError,
+ ProtocolError,
+ BaseSSLError,
+ SSLError,
+ CertificateError,
+ ProxyError,
+ ) as e:
+ # Discard the connection for these exceptions. It will be
+ # replaced during the next _get_conn() call.
+ clean_exit = False
+ new_e: Exception = e
+ if isinstance(e, (BaseSSLError, CertificateError)):
+ new_e = SSLError(e)
+ if isinstance(
+ new_e,
+ (
+ OSError,
+ NewConnectionError,
+ TimeoutError,
+ SSLError,
+ HTTPException,
+ ),
+ ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
+ new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
+ elif isinstance(new_e, (OSError, HTTPException)):
+ new_e = ProtocolError("Connection aborted.", new_e)
+
+ retries = retries.increment(
+ method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
+ )
+ retries.sleep()
+
+ # Keep track of the error for the retry warning.
+ err = e
+
+ finally:
+ if not clean_exit:
+ # We hit some kind of exception, handled or otherwise. We need
+ # to throw the connection away unless explicitly told not to.
+ # Close the connection, set the variable to None, and make sure
+ # we put the None back in the pool to avoid leaking it.
+ if conn:
+ conn.close()
+ conn = None
+ release_this_conn = True
+
+ if release_this_conn:
+ # Put the connection back to be reused. If the connection is
+ # expired then it will be None, which will get replaced with a
+ # fresh connection during _get_conn.
+ self._put_conn(conn)
+
+ if not conn:
+ # Try again
+ log.warning(
+ "Retrying (%r) after connection broken by '%r': %s", retries, err, url
+ )
+ return self.urlopen(
+ method,
+ url,
+ body,
+ headers,
+ retries,
+ redirect,
+ assert_same_host,
+ timeout=timeout,
+ pool_timeout=pool_timeout,
+ release_conn=release_conn,
+ chunked=chunked,
+ body_pos=body_pos,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ **response_kw,
+ )
+
+ # Handle redirect?
+ redirect_location = redirect and response.get_redirect_location()
+ if redirect_location:
+ if response.status == 303:
+ # Change the method according to RFC 9110, Section 15.4.4.
+ method = "GET"
+ # And lose the body not to transfer anything sensitive.
+ body = None
+ headers = HTTPHeaderDict(headers)._prepare_for_method_change()
+
+ try:
+ retries = retries.increment(method, url, response=response, _pool=self)
+ except MaxRetryError:
+ if retries.raise_on_redirect:
+ response.drain_conn()
+ raise
+ return response
+
+ response.drain_conn()
+ retries.sleep_for_retry(response)
+ log.debug("Redirecting %s -> %s", url, redirect_location)
+ return self.urlopen(
+ method,
+ redirect_location,
+ body,
+ headers,
+ retries=retries,
+ redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout,
+ pool_timeout=pool_timeout,
+ release_conn=release_conn,
+ chunked=chunked,
+ body_pos=body_pos,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ **response_kw,
+ )
+
+ # Check if we should retry the HTTP response.
+ has_retry_after = bool(response.headers.get("Retry-After"))
+ if retries.is_retry(method, response.status, has_retry_after):
+ try:
+ retries = retries.increment(method, url, response=response, _pool=self)
+ except MaxRetryError:
+ if retries.raise_on_status:
+ response.drain_conn()
+ raise
+ return response
+
+ response.drain_conn()
+ retries.sleep(response)
+ log.debug("Retry: %s", url)
+ return self.urlopen(
+ method,
+ url,
+ body,
+ headers,
+ retries=retries,
+ redirect=redirect,
+ assert_same_host=assert_same_host,
+ timeout=timeout,
+ pool_timeout=pool_timeout,
+ release_conn=release_conn,
+ chunked=chunked,
+ body_pos=body_pos,
+ preload_content=preload_content,
+ decode_content=decode_content,
+ **response_kw,
+ )
+
+ return response
+
+
+class HTTPSConnectionPool(HTTPConnectionPool):
+ """
+ Same as :class:`.HTTPConnectionPool`, but HTTPS.
+
+ :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
+ ``assert_hostname`` and ``host`` in this order to verify connections.
+ If ``assert_hostname`` is False, no verification is done.
+
+ The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
+ ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
+ is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
+ the connection socket into an SSL socket.
+ """
+
+ scheme = "https"
+ ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
+
+ def __init__(
+ self,
+ host: str,
+ port: int | None = None,
+ timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
+ maxsize: int = 1,
+ block: bool = False,
+ headers: typing.Mapping[str, str] | None = None,
+ retries: Retry | bool | int | None = None,
+ _proxy: Url | None = None,
+ _proxy_headers: typing.Mapping[str, str] | None = None,
+ key_file: str | None = None,
+ cert_file: str | None = None,
+ cert_reqs: int | str | None = None,
+ key_password: str | None = None,
+ ca_certs: str | None = None,
+ ssl_version: int | str | None = None,
+ ssl_minimum_version: ssl.TLSVersion | None = None,
+ ssl_maximum_version: ssl.TLSVersion | None = None,
+ assert_hostname: str | typing.Literal[False] | None = None,
+ assert_fingerprint: str | None = None,
+ ca_cert_dir: str | None = None,
+ **conn_kw: typing.Any,
+ ) -> None:
+ super().__init__(
+ host,
+ port,
+ timeout,
+ maxsize,
+ block,
+ headers,
+ retries,
+ _proxy,
+ _proxy_headers,
+ **conn_kw,
+ )
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.cert_reqs = cert_reqs
+ self.key_password = key_password
+ self.ca_certs = ca_certs
+ self.ca_cert_dir = ca_cert_dir
+ self.ssl_version = ssl_version
+ self.ssl_minimum_version = ssl_minimum_version
+ self.ssl_maximum_version = ssl_maximum_version
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+
+ def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
+ """Establishes a tunnel connection through HTTP CONNECT."""
+ if self.proxy and self.proxy.scheme == "https":
+ tunnel_scheme = "https"
+ else:
+ tunnel_scheme = "http"
+
+ conn.set_tunnel(
+ scheme=tunnel_scheme,
+ host=self._tunnel_host,
+ port=self.port,
+ headers=self.proxy_headers,
+ )
+ conn.connect()
+
+ def _new_conn(self) -> BaseHTTPSConnection:
+ """
+ Return a fresh :class:`urllib3.connection.HTTPConnection`.
+ """
+ self.num_connections += 1
+ log.debug(
+ "Starting new HTTPS connection (%d): %s:%s",
+ self.num_connections,
+ self.host,
+ self.port or "443",
+ )
+
+ if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
+ raise ImportError(
+ "Can't connect to HTTPS URL because the SSL module is not available."
+ )
+
+ actual_host: str = self.host
+ actual_port = self.port
+ if self.proxy is not None and self.proxy.host is not None:
+ actual_host = self.proxy.host
+ actual_port = self.proxy.port
+
+ return self.ConnectionCls(
+ host=actual_host,
+ port=actual_port,
+ timeout=self.timeout.connect_timeout,
+ cert_file=self.cert_file,
+ key_file=self.key_file,
+ key_password=self.key_password,
+ cert_reqs=self.cert_reqs,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ assert_hostname=self.assert_hostname,
+ assert_fingerprint=self.assert_fingerprint,
+ ssl_version=self.ssl_version,
+ ssl_minimum_version=self.ssl_minimum_version,
+ ssl_maximum_version=self.ssl_maximum_version,
+ **self.conn_kw,
+ )
+
+ def _validate_conn(self, conn: BaseHTTPConnection) -> None:
+ """
+ Called right before a request is made, after the socket is created.
+ """
+ super()._validate_conn(conn)
+
+ # Force connect early to allow us to validate the connection.
+ if conn.is_closed:
+ conn.connect()
+
+ # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
+ if not conn.is_verified and not conn.proxy_is_verified:
+ warnings.warn(
+ (
+ f"Unverified HTTPS request is being made to host '{conn.host}'. "
+ "Adding certificate verification is strongly advised. See: "
+ "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
+ "#tls-warnings"
+ ),
+ InsecureRequestWarning,
+ )
+
+
+def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
+ """
+ Given a url, return an :class:`.ConnectionPool` instance of its host.
+
+ This is a shortcut for not having to parse out the scheme, host, and port
+ of the url before creating an :class:`.ConnectionPool` instance.
+
+ :param url:
+ Absolute URL string that must include the scheme. Port is optional.
+
+ :param \\**kw:
+ Passes additional parameters to the constructor of the appropriate
+ :class:`.ConnectionPool`. Useful for specifying things like
+ timeout, maxsize, headers, etc.
+
+ Example::
+
+ >>> conn = connection_from_url('http://google.com/')
+ >>> r = conn.request('GET', '/')
+ """
+ scheme, _, host, port, *_ = parse_url(url)
+ scheme = scheme or "http"
+ port = port or port_by_scheme.get(scheme, 80)
+ if scheme == "https":
+ return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
+ else:
+ return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
+
+
+@typing.overload
+def _normalize_host(host: None, scheme: str | None) -> None: ...
+
+
+@typing.overload
+def _normalize_host(host: str, scheme: str | None) -> str: ...
+
+
+def _normalize_host(host: str | None, scheme: str | None) -> str | None:
+ """
+ Normalize hosts for comparisons and use with sockets.
+ """
+
+ host = normalize_host(host, scheme)
+
+ # httplib doesn't like it when we include brackets in IPv6 addresses
+ # Specifically, if we include brackets but also pass the port then
+ # httplib crazily doubles up the square brackets on the Host header.
+ # Instead, we need to make sure we never pass ``None`` as the port.
+ # However, for backward compatibility reasons we can't actually
+ # *assert* that. See http://bugs.python.org/issue28539
+ if host and host.startswith("[") and host.endswith("]"):
+ host = host[1:-1]
+ return host
+
+
+def _url_from_pool(
+ pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
+) -> str:
+ """Returns the URL from a given connection pool. This is mainly used for testing and logging."""
+ return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
+
+
+def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
+ """Drains a queue of connections and closes each one."""
+ try:
+ while True:
+ conn = pool.get(block=False)
+ if conn:
+ conn.close()
+ except queue.Empty:
+ pass # Done.
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/__init__.py b/phivenv/Lib/site-packages/urllib3/contrib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ac23037232dc2191847c53a1c038307103c6838
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/pyopenssl.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/pyopenssl.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..308f3fbdebd00fae6ca99acf1854856b63678608
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/pyopenssl.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/socks.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/socks.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7b729f9d5e0100abc05c76bf027bfbf4ff7931c5
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/__pycache__/socks.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__init__.py b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a3c5bebdc151eef715663628a697118bb2932ed
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__init__.py
@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import urllib3.connection
+
+from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
+
+
+def inject_into_urllib3() -> None:
+ # override connection classes to use emscripten specific classes
+ # n.b. mypy complains about the overriding of classes below
+ # if it isn't ignored
+ HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
+ HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
+ urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment]
+ urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment]
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..835dab5c91635e3eb4a7901463012d0148be65ee
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/connection.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/connection.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d1cae1cacd0bb8bd7b5919483ea28da8f7552d85
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/connection.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/fetch.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/fetch.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6db6d742a40a901d8d3e0c637ac51fe44f4d76bc
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/fetch.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/request.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/request.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..063e58a2686bbbfe9f097d02e80d3364b9598bd4
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/request.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/response.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/response.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..143a130c182969791b9e2589978c5286b4e5cd07
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/__pycache__/response.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/connection.py b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..41bfd2797ffeef54fa50f708026c4cc44b7d2c12
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/connection.py
@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import os
+import typing
+
+# use http.client.HTTPException for consistency with non-emscripten
+from http.client import HTTPException as HTTPException # noqa: F401
+from http.client import ResponseNotReady
+
+from ..._base_connection import _TYPE_BODY
+from ...connection import HTTPConnection, ProxyConfig, port_by_scheme
+from ...exceptions import TimeoutError
+from ...response import BaseHTTPResponse
+from ...util.connection import _TYPE_SOCKET_OPTIONS
+from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
+from ...util.url import Url
+from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request
+from .request import EmscriptenRequest
+from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse
+
+if typing.TYPE_CHECKING:
+ from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
+
+
+class EmscriptenHTTPConnection:
+ default_port: typing.ClassVar[int] = port_by_scheme["http"]
+ default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
+
+ timeout: None | (float)
+
+ host: str
+ port: int
+ blocksize: int
+ source_address: tuple[str, int] | None
+ socket_options: _TYPE_SOCKET_OPTIONS | None
+
+ proxy: Url | None
+ proxy_config: ProxyConfig | None
+
+ is_verified: bool = False
+ proxy_is_verified: bool | None = None
+
+ _response: EmscriptenResponse | None
+
+ def __init__(
+ self,
+ host: str,
+ port: int = 0,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 8192,
+ socket_options: _TYPE_SOCKET_OPTIONS | None = None,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ ) -> None:
+ self.host = host
+ self.port = port
+ self.timeout = timeout if isinstance(timeout, float) else 0.0
+ self.scheme = "http"
+ self._closed = True
+ self._response = None
+ # ignore these things because we don't
+ # have control over that stuff
+ self.proxy = None
+ self.proxy_config = None
+ self.blocksize = blocksize
+ self.source_address = None
+ self.socket_options = None
+ self.is_verified = False
+
+ def set_tunnel(
+ self,
+ host: str,
+ port: int | None = 0,
+ headers: typing.Mapping[str, str] | None = None,
+ scheme: str = "http",
+ ) -> None:
+ pass
+
+ def connect(self) -> None:
+ pass
+
+ def request(
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ # We know *at least* botocore is depending on the order of the
+ # first 3 parameters so to be safe we only mark the later ones
+ # as keyword-only to ensure we have space to extend.
+ *,
+ chunked: bool = False,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ enforce_content_length: bool = True,
+ ) -> None:
+ self._closed = False
+ if url.startswith("/"):
+ # no scheme / host / port included, make a full url
+ url = f"{self.scheme}://{self.host}:{self.port}" + url
+ request = EmscriptenRequest(
+ url=url,
+ method=method,
+ timeout=self.timeout if self.timeout else 0,
+ decode_content=decode_content,
+ )
+ request.set_body(body)
+ if headers:
+ for k, v in headers.items():
+ request.set_header(k, v)
+ self._response = None
+ try:
+ if not preload_content:
+ self._response = send_streaming_request(request)
+ if self._response is None:
+ self._response = send_request(request)
+ except _TimeoutError as e:
+ raise TimeoutError(e.message) from e
+ except _RequestError as e:
+ raise HTTPException(e.message) from e
+
+ def getresponse(self) -> BaseHTTPResponse:
+ if self._response is not None:
+ return EmscriptenHttpResponseWrapper(
+ internal_response=self._response,
+ url=self._response.request.url,
+ connection=self,
+ )
+ else:
+ raise ResponseNotReady()
+
+ def close(self) -> None:
+ self._closed = True
+ self._response = None
+
+ @property
+ def is_closed(self) -> bool:
+ """Whether the connection either is brand new or has been previously closed.
+ If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
+ properties must be False.
+ """
+ return self._closed
+
+ @property
+ def is_connected(self) -> bool:
+ """Whether the connection is actively connected to any origin (proxy or target)"""
+ return True
+
+ @property
+ def has_connected_to_proxy(self) -> bool:
+ """Whether the connection has successfully connected to its proxy.
+ This returns False if no proxy is in use. Used to determine whether
+ errors are coming from the proxy layer or from tunnelling to the target origin.
+ """
+ return False
+
+
+class EmscriptenHTTPSConnection(EmscriptenHTTPConnection):
+ default_port = port_by_scheme["https"]
+ # all this is basically ignored, as browser handles https
+ cert_reqs: int | str | None = None
+ ca_certs: str | None = None
+ ca_cert_dir: str | None = None
+ ca_cert_data: None | str | bytes = None
+ cert_file: str | None
+ key_file: str | None
+ key_password: str | None
+ ssl_context: typing.Any | None
+ ssl_version: int | str | None = None
+ ssl_minimum_version: int | None = None
+ ssl_maximum_version: int | None = None
+ assert_hostname: None | str | typing.Literal[False]
+ assert_fingerprint: str | None = None
+
+ def __init__(
+ self,
+ host: str,
+ port: int = 0,
+ *,
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ blocksize: int = 16384,
+ socket_options: (
+ None | _TYPE_SOCKET_OPTIONS
+ ) = HTTPConnection.default_socket_options,
+ proxy: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ cert_reqs: int | str | None = None,
+ assert_hostname: None | str | typing.Literal[False] = None,
+ assert_fingerprint: str | None = None,
+ server_hostname: str | None = None,
+ ssl_context: typing.Any | None = None,
+ ca_certs: str | None = None,
+ ca_cert_dir: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ ssl_minimum_version: int | None = None,
+ ssl_maximum_version: int | None = None,
+ ssl_version: int | str | None = None, # Deprecated
+ cert_file: str | None = None,
+ key_file: str | None = None,
+ key_password: str | None = None,
+ ) -> None:
+ super().__init__(
+ host,
+ port=port,
+ timeout=timeout,
+ source_address=source_address,
+ blocksize=blocksize,
+ socket_options=socket_options,
+ proxy=proxy,
+ proxy_config=proxy_config,
+ )
+ self.scheme = "https"
+
+ self.key_file = key_file
+ self.cert_file = cert_file
+ self.key_password = key_password
+ self.ssl_context = ssl_context
+ self.server_hostname = server_hostname
+ self.assert_hostname = assert_hostname
+ self.assert_fingerprint = assert_fingerprint
+ self.ssl_version = ssl_version
+ self.ssl_minimum_version = ssl_minimum_version
+ self.ssl_maximum_version = ssl_maximum_version
+ self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
+ self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
+ self.ca_cert_data = ca_cert_data
+
+ self.cert_reqs = None
+
+ # The browser will automatically verify all requests.
+ # We have no control over that setting.
+ self.is_verified = True
+
+ def set_cert(
+ self,
+ key_file: str | None = None,
+ cert_file: str | None = None,
+ cert_reqs: int | str | None = None,
+ key_password: str | None = None,
+ ca_certs: str | None = None,
+ assert_hostname: None | str | typing.Literal[False] = None,
+ assert_fingerprint: str | None = None,
+ ca_cert_dir: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ ) -> None:
+ pass
+
+
+# verify that this class implements BaseHTTP(s) connection correctly
+if typing.TYPE_CHECKING:
+ _supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0)
+ _supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0)
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/emscripten_fetch_worker.js b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/emscripten_fetch_worker.js
new file mode 100644
index 0000000000000000000000000000000000000000..243b86222f90a9be4b6b4ce0bf997eefd29289af
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/emscripten_fetch_worker.js
@@ -0,0 +1,110 @@
+let Status = {
+ SUCCESS_HEADER: -1,
+ SUCCESS_EOF: -2,
+ ERROR_TIMEOUT: -3,
+ ERROR_EXCEPTION: -4,
+};
+
+let connections = {};
+let nextConnectionID = 1;
+const encoder = new TextEncoder();
+
+self.addEventListener("message", async function (event) {
+ if (event.data.close) {
+ let connectionID = event.data.close;
+ delete connections[connectionID];
+ return;
+ } else if (event.data.getMore) {
+ let connectionID = event.data.getMore;
+ let { curOffset, value, reader, intBuffer, byteBuffer } =
+ connections[connectionID];
+ // if we still have some in buffer, then just send it back straight away
+ if (!value || curOffset >= value.length) {
+ // read another buffer if required
+ try {
+ let readResponse = await reader.read();
+
+ if (readResponse.done) {
+ // read everything - clear connection and return
+ delete connections[connectionID];
+ Atomics.store(intBuffer, 0, Status.SUCCESS_EOF);
+ Atomics.notify(intBuffer, 0);
+ // finished reading successfully
+ // return from event handler
+ return;
+ }
+ curOffset = 0;
+ connections[connectionID].value = readResponse.value;
+ value = readResponse.value;
+ } catch (error) {
+ console.log("Request exception:", error);
+ let errorBytes = encoder.encode(error.message);
+ let written = errorBytes.length;
+ byteBuffer.set(errorBytes);
+ intBuffer[1] = written;
+ Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
+ Atomics.notify(intBuffer, 0);
+ }
+ }
+
+ // send as much buffer as we can
+ let curLen = value.length - curOffset;
+ if (curLen > byteBuffer.length) {
+ curLen = byteBuffer.length;
+ }
+ byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0);
+
+ Atomics.store(intBuffer, 0, curLen); // store current length in bytes
+ Atomics.notify(intBuffer, 0);
+ curOffset += curLen;
+ connections[connectionID].curOffset = curOffset;
+
+ return;
+ } else {
+ // start fetch
+ let connectionID = nextConnectionID;
+ nextConnectionID += 1;
+ const intBuffer = new Int32Array(event.data.buffer);
+ const byteBuffer = new Uint8Array(event.data.buffer, 8);
+ try {
+ const response = await fetch(event.data.url, event.data.fetchParams);
+ // return the headers first via textencoder
+ var headers = [];
+ for (const pair of response.headers.entries()) {
+ headers.push([pair[0], pair[1]]);
+ }
+ let headerObj = {
+ headers: headers,
+ status: response.status,
+ connectionID,
+ };
+ const headerText = JSON.stringify(headerObj);
+ let headerBytes = encoder.encode(headerText);
+ let written = headerBytes.length;
+ byteBuffer.set(headerBytes);
+ intBuffer[1] = written;
+ // make a connection
+ connections[connectionID] = {
+ reader: response.body.getReader(),
+ intBuffer: intBuffer,
+ byteBuffer: byteBuffer,
+ value: undefined,
+ curOffset: 0,
+ };
+ // set header ready
+ Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER);
+ Atomics.notify(intBuffer, 0);
+ // all fetching after this goes through a new postmessage call with getMore
+ // this allows for parallel requests
+ } catch (error) {
+ console.log("Request exception:", error);
+ let errorBytes = encoder.encode(error.message);
+ let written = errorBytes.length;
+ byteBuffer.set(errorBytes);
+ intBuffer[1] = written;
+ Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
+ Atomics.notify(intBuffer, 0);
+ }
+ }
+});
+self.postMessage({ inited: true });
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/fetch.py b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/fetch.py
new file mode 100644
index 0000000000000000000000000000000000000000..66958217a9fdc8a03eb991d6da2558d7e7910150
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/fetch.py
@@ -0,0 +1,728 @@
+"""
+Support for streaming http requests in emscripten.
+
+A few caveats -
+
+If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled
+https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md
+*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the
+JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case
+timeouts and streaming should just work.
+
+Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming.
+
+This approach has several caveats:
+
+Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
+Streaming only works if you're running pyodide in a web worker.
+
+Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
+operation, so it requires that you have crossOriginIsolation enabled, by serving over https
+(or from localhost) with the two headers below set:
+
+ Cross-Origin-Opener-Policy: same-origin
+ Cross-Origin-Embedder-Policy: require-corp
+
+You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
+JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
+request into a buffer and then returning it. it shows a warning in the JavaScript console in this case.
+
+Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
+control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
+
+NB: in this code, there are a lot of JavaScript objects. They are named js_*
+to make it clear what type of object they are.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+from email.parser import Parser
+from importlib.resources import files
+from typing import TYPE_CHECKING, Any
+
+import js # type: ignore[import-not-found]
+from pyodide.ffi import ( # type: ignore[import-not-found]
+ JsArray,
+ JsException,
+ JsProxy,
+ to_js,
+)
+
+if TYPE_CHECKING:
+ from typing_extensions import Buffer
+
+from .request import EmscriptenRequest
+from .response import EmscriptenResponse
+
+"""
+There are some headers that trigger unintended CORS preflight requests.
+See also https://github.com/koenvo/pyodide-http/issues/22
+"""
+HEADERS_TO_IGNORE = ("user-agent",)
+
+SUCCESS_HEADER = -1
+SUCCESS_EOF = -2
+ERROR_TIMEOUT = -3
+ERROR_EXCEPTION = -4
+
+_STREAMING_WORKER_CODE = (
+ files(__package__)
+ .joinpath("emscripten_fetch_worker.js")
+ .read_text(encoding="utf-8")
+)
+
+
+class _RequestError(Exception):
+ def __init__(
+ self,
+ message: str | None = None,
+ *,
+ request: EmscriptenRequest | None = None,
+ response: EmscriptenResponse | None = None,
+ ):
+ self.request = request
+ self.response = response
+ self.message = message
+ super().__init__(self.message)
+
+
+class _StreamingError(_RequestError):
+ pass
+
+
+class _TimeoutError(_RequestError):
+ pass
+
+
+def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy:
+ return to_js(dict_val, dict_converter=js.Object.fromEntries)
+
+
+class _ReadStream(io.RawIOBase):
+ def __init__(
+ self,
+ int_buffer: JsArray,
+ byte_buffer: JsArray,
+ timeout: float,
+ worker: JsProxy,
+ connection_id: int,
+ request: EmscriptenRequest,
+ ):
+ self.int_buffer = int_buffer
+ self.byte_buffer = byte_buffer
+ self.read_pos = 0
+ self.read_len = 0
+ self.connection_id = connection_id
+ self.worker = worker
+ self.timeout = int(1000 * timeout) if timeout > 0 else None
+ self.is_live = True
+ self._is_closed = False
+ self.request: EmscriptenRequest | None = request
+
+ def __del__(self) -> None:
+ self.close()
+
+ # this is compatible with _base_connection
+ def is_closed(self) -> bool:
+ return self._is_closed
+
+ # for compatibility with RawIOBase
+ @property
+ def closed(self) -> bool:
+ return self.is_closed()
+
+ def close(self) -> None:
+ if self.is_closed():
+ return
+ self.read_len = 0
+ self.read_pos = 0
+ self.int_buffer = None
+ self.byte_buffer = None
+ self._is_closed = True
+ self.request = None
+ if self.is_live:
+ self.worker.postMessage(_obj_from_dict({"close": self.connection_id}))
+ self.is_live = False
+ super().close()
+
+ def readable(self) -> bool:
+ return True
+
+ def writable(self) -> bool:
+ return False
+
+ def seekable(self) -> bool:
+ return False
+
+ def readinto(self, byte_obj: Buffer) -> int:
+ if not self.int_buffer:
+ raise _StreamingError(
+ "No buffer for stream in _ReadStream.readinto",
+ request=self.request,
+ response=None,
+ )
+ if self.read_len == 0:
+ # wait for the worker to send something
+ js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT)
+ self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id}))
+ if (
+ js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout)
+ == "timed-out"
+ ):
+ raise _TimeoutError
+ data_len = self.int_buffer[0]
+ if data_len > 0:
+ self.read_len = data_len
+ self.read_pos = 0
+ elif data_len == ERROR_EXCEPTION:
+ string_len = self.int_buffer[1]
+ # decode the error string
+ js_decoder = js.TextDecoder.new()
+ json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len))
+ raise _StreamingError(
+ f"Exception thrown in fetch: {json_str}",
+ request=self.request,
+ response=None,
+ )
+ else:
+ # EOF, free the buffers and return zero
+ # and free the request
+ self.is_live = False
+ self.close()
+ return 0
+ # copy from int32array to python bytes
+ ret_length = min(self.read_len, len(memoryview(byte_obj)))
+ subarray = self.byte_buffer.subarray(
+ self.read_pos, self.read_pos + ret_length
+ ).to_py()
+ memoryview(byte_obj)[0:ret_length] = subarray
+ self.read_len -= ret_length
+ self.read_pos += ret_length
+ return ret_length
+
+
+class _StreamingFetcher:
+ def __init__(self) -> None:
+ # make web-worker and data buffer on startup
+ self.streaming_ready = False
+
+ js_data_blob = js.Blob.new(
+ to_js([_STREAMING_WORKER_CODE], create_pyproxies=False),
+ _obj_from_dict({"type": "application/javascript"}),
+ )
+
+ def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None:
+ def onMsg(e: JsProxy) -> None:
+ self.streaming_ready = True
+ js_resolve_fn(e)
+
+ def onErr(e: JsProxy) -> None:
+ js_reject_fn(e) # Defensive: never happens in ci
+
+ self.js_worker.onmessage = onMsg
+ self.js_worker.onerror = onErr
+
+ js_data_url = js.URL.createObjectURL(js_data_blob)
+ self.js_worker = js.globalThis.Worker.new(js_data_url)
+ self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver)
+
+ def send(self, request: EmscriptenRequest) -> EmscriptenResponse:
+ headers = {
+ k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE
+ }
+
+ body = request.body
+ fetch_data = {"headers": headers, "body": to_js(body), "method": request.method}
+ # start the request off in the worker
+ timeout = int(1000 * request.timeout) if request.timeout > 0 else None
+ js_shared_buffer = js.SharedArrayBuffer.new(1048576)
+ js_int_buffer = js.Int32Array.new(js_shared_buffer)
+ js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8)
+
+ js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT)
+ js.Atomics.notify(js_int_buffer, 0)
+ js_absolute_url = js.URL.new(request.url, js.location).href
+ self.js_worker.postMessage(
+ _obj_from_dict(
+ {
+ "buffer": js_shared_buffer,
+ "url": js_absolute_url,
+ "fetchParams": fetch_data,
+ }
+ )
+ )
+ # wait for the worker to send something
+ js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout)
+ if js_int_buffer[0] == ERROR_TIMEOUT:
+ raise _TimeoutError(
+ "Timeout connecting to streaming request",
+ request=request,
+ response=None,
+ )
+ elif js_int_buffer[0] == SUCCESS_HEADER:
+ # got response
+ # header length is in second int of intBuffer
+ string_len = js_int_buffer[1]
+ # decode the rest to a JSON string
+ js_decoder = js.TextDecoder.new()
+ # this does a copy (the slice) because decode can't work on shared array
+ # for some silly reason
+ json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
+ # get it as an object
+ response_obj = json.loads(json_str)
+ return EmscriptenResponse(
+ request=request,
+ status_code=response_obj["status"],
+ headers=response_obj["headers"],
+ body=_ReadStream(
+ js_int_buffer,
+ js_byte_buffer,
+ request.timeout,
+ self.js_worker,
+ response_obj["connectionID"],
+ request,
+ ),
+ )
+ elif js_int_buffer[0] == ERROR_EXCEPTION:
+ string_len = js_int_buffer[1]
+ # decode the error string
+ js_decoder = js.TextDecoder.new()
+ json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
+ raise _StreamingError(
+ f"Exception thrown in fetch: {json_str}", request=request, response=None
+ )
+ else:
+ raise _StreamingError(
+ f"Unknown status from worker in fetch: {js_int_buffer[0]}",
+ request=request,
+ response=None,
+ )
+
+
+class _JSPIReadStream(io.RawIOBase):
+ """
+ A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch
+ response. This requires support for WebAssembly JavaScript Promise Integration
+ in the containing browser, and for pyodide to be launched via runPythonAsync.
+
+ :param js_read_stream:
+ The JavaScript stream reader
+
+ :param timeout:
+ Timeout in seconds
+
+ :param request:
+ The request we're handling
+
+ :param response:
+ The response this stream relates to
+
+ :param js_abort_controller:
+ A JavaScript AbortController object, used for timeouts
+ """
+
+ def __init__(
+ self,
+ js_read_stream: Any,
+ timeout: float,
+ request: EmscriptenRequest,
+ response: EmscriptenResponse,
+ js_abort_controller: Any, # JavaScript AbortController for timeouts
+ ):
+ self.js_read_stream = js_read_stream
+ self.timeout = timeout
+ self._is_closed = False
+ self._is_done = False
+ self.request: EmscriptenRequest | None = request
+ self.response: EmscriptenResponse | None = response
+ self.current_buffer = None
+ self.current_buffer_pos = 0
+ self.js_abort_controller = js_abort_controller
+
+ def __del__(self) -> None:
+ self.close()
+
+ # this is compatible with _base_connection
+ def is_closed(self) -> bool:
+ return self._is_closed
+
+ # for compatibility with RawIOBase
+ @property
+ def closed(self) -> bool:
+ return self.is_closed()
+
+ def close(self) -> None:
+ if self.is_closed():
+ return
+ self.read_len = 0
+ self.read_pos = 0
+ self.js_read_stream.cancel()
+ self.js_read_stream = None
+ self._is_closed = True
+ self._is_done = True
+ self.request = None
+ self.response = None
+ super().close()
+
+ def readable(self) -> bool:
+ return True
+
+ def writable(self) -> bool:
+ return False
+
+ def seekable(self) -> bool:
+ return False
+
+ def _get_next_buffer(self) -> bool:
+ result_js = _run_sync_with_timeout(
+ self.js_read_stream.read(),
+ self.timeout,
+ self.js_abort_controller,
+ request=self.request,
+ response=self.response,
+ )
+ if result_js.done:
+ self._is_done = True
+ return False
+ else:
+ self.current_buffer = result_js.value.to_py()
+ self.current_buffer_pos = 0
+ return True
+
+ def readinto(self, byte_obj: Buffer) -> int:
+ if self.current_buffer is None:
+ if not self._get_next_buffer() or self.current_buffer is None:
+ self.close()
+ return 0
+ ret_length = min(
+ len(byte_obj), len(self.current_buffer) - self.current_buffer_pos
+ )
+ byte_obj[0:ret_length] = self.current_buffer[
+ self.current_buffer_pos : self.current_buffer_pos + ret_length
+ ]
+ self.current_buffer_pos += ret_length
+ if self.current_buffer_pos == len(self.current_buffer):
+ self.current_buffer = None
+ return ret_length
+
+
+# check if we are in a worker or not
+def is_in_browser_main_thread() -> bool:
+ return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window
+
+
+def is_cross_origin_isolated() -> bool:
+ return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated
+
+
+def is_in_node() -> bool:
+ return (
+ hasattr(js, "process")
+ and hasattr(js.process, "release")
+ and hasattr(js.process.release, "name")
+ and js.process.release.name == "node"
+ )
+
+
+def is_worker_available() -> bool:
+ return hasattr(js, "Worker") and hasattr(js, "Blob")
+
+
+_fetcher: _StreamingFetcher | None = None
+
+if is_worker_available() and (
+ (is_cross_origin_isolated() and not is_in_browser_main_thread())
+ and (not is_in_node())
+):
+ _fetcher = _StreamingFetcher()
+else:
+ _fetcher = None
+
+
+NODE_JSPI_ERROR = (
+ "urllib3 only works in Node.js with pyodide.runPythonAsync"
+ " and requires the flag --experimental-wasm-stack-switching in "
+ " versions of node <24."
+)
+
+
+def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None:
+ if has_jspi():
+ return send_jspi_request(request, True)
+ elif is_in_node():
+ raise _RequestError(
+ message=NODE_JSPI_ERROR,
+ request=request,
+ response=None,
+ )
+
+ if _fetcher and streaming_ready():
+ return _fetcher.send(request)
+ else:
+ _show_streaming_warning()
+ return None
+
+
+_SHOWN_TIMEOUT_WARNING = False
+
+
+def _show_timeout_warning() -> None:
+ global _SHOWN_TIMEOUT_WARNING
+ if not _SHOWN_TIMEOUT_WARNING:
+ _SHOWN_TIMEOUT_WARNING = True
+ message = "Warning: Timeout is not available on main browser thread"
+ js.console.warn(message)
+
+
+_SHOWN_STREAMING_WARNING = False
+
+
+def _show_streaming_warning() -> None:
+ global _SHOWN_STREAMING_WARNING
+ if not _SHOWN_STREAMING_WARNING:
+ _SHOWN_STREAMING_WARNING = True
+ message = "Can't stream HTTP requests because: \n"
+ if not is_cross_origin_isolated():
+ message += " Page is not cross-origin isolated\n"
+ if is_in_browser_main_thread():
+ message += " Python is running in main browser thread\n"
+ if not is_worker_available():
+ message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in
+ if streaming_ready() is False:
+ message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
+is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`"""
+ from js import console
+
+ console.warn(message)
+
+
+def send_request(request: EmscriptenRequest) -> EmscriptenResponse:
+ if has_jspi():
+ return send_jspi_request(request, False)
+ elif is_in_node():
+ raise _RequestError(
+ message=NODE_JSPI_ERROR,
+ request=request,
+ response=None,
+ )
+ try:
+ js_xhr = js.XMLHttpRequest.new()
+
+ if not is_in_browser_main_thread():
+ js_xhr.responseType = "arraybuffer"
+ if request.timeout:
+ js_xhr.timeout = int(request.timeout * 1000)
+ else:
+ js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15")
+ if request.timeout:
+ # timeout isn't available on the main thread - show a warning in console
+ # if it is set
+ _show_timeout_warning()
+
+ js_xhr.open(request.method, request.url, False)
+ for name, value in request.headers.items():
+ if name.lower() not in HEADERS_TO_IGNORE:
+ js_xhr.setRequestHeader(name, value)
+
+ js_xhr.send(to_js(request.body))
+
+ headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders()))
+
+ if not is_in_browser_main_thread():
+ body = js_xhr.response.to_py().tobytes()
+ else:
+ body = js_xhr.response.encode("ISO-8859-15")
+ return EmscriptenResponse(
+ status_code=js_xhr.status, headers=headers, body=body, request=request
+ )
+ except JsException as err:
+ if err.name == "TimeoutError":
+ raise _TimeoutError(err.message, request=request)
+ elif err.name == "NetworkError":
+ raise _RequestError(err.message, request=request)
+ else:
+ # general http error
+ raise _RequestError(err.message, request=request)
+
+
+def send_jspi_request(
+ request: EmscriptenRequest, streaming: bool
+) -> EmscriptenResponse:
+ """
+ Send a request using WebAssembly JavaScript Promise Integration
+ to wrap the asynchronous JavaScript fetch api (experimental).
+
+ :param request:
+ Request to send
+
+ :param streaming:
+ Whether to stream the response
+
+ :return: The response object
+ :rtype: EmscriptenResponse
+ """
+ timeout = request.timeout
+ js_abort_controller = js.AbortController.new()
+ headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE}
+ req_body = request.body
+ fetch_data = {
+ "headers": headers,
+ "body": to_js(req_body),
+ "method": request.method,
+ "signal": js_abort_controller.signal,
+ }
+ # Node.js returns the whole response (unlike opaqueredirect in browsers),
+ # so urllib3 can set `redirect: manual` to control redirects itself.
+ # https://stackoverflow.com/a/78524615
+ if _is_node_js():
+ fetch_data["redirect"] = "manual"
+ # Call JavaScript fetch (async api, returns a promise)
+ fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data))
+ # Now suspend WebAssembly until we resolve that promise
+ # or time out.
+ response_js = _run_sync_with_timeout(
+ fetcher_promise_js,
+ timeout,
+ js_abort_controller,
+ request=request,
+ response=None,
+ )
+ headers = {}
+ header_iter = response_js.headers.entries()
+ while True:
+ iter_value_js = header_iter.next()
+ if getattr(iter_value_js, "done", False):
+ break
+ else:
+ headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1])
+ status_code = response_js.status
+ body: bytes | io.RawIOBase = b""
+
+ response = EmscriptenResponse(
+ status_code=status_code, headers=headers, body=b"", request=request
+ )
+ if streaming:
+ # get via inputstream
+ if response_js.body is not None:
+ # get a reader from the fetch response
+ body_stream_js = response_js.body.getReader()
+ body = _JSPIReadStream(
+ body_stream_js, timeout, request, response, js_abort_controller
+ )
+ else:
+ # get directly via arraybuffer
+ # n.b. this is another async JavaScript call.
+ body = _run_sync_with_timeout(
+ response_js.arrayBuffer(),
+ timeout,
+ js_abort_controller,
+ request=request,
+ response=response,
+ ).to_py()
+ response.body = body
+ return response
+
+
+def _run_sync_with_timeout(
+ promise: Any,
+ timeout: float,
+ js_abort_controller: Any,
+ request: EmscriptenRequest | None,
+ response: EmscriptenResponse | None,
+) -> Any:
+ """
+ Await a JavaScript promise synchronously with a timeout which is implemented
+ via the AbortController
+
+ :param promise:
+ Javascript promise to await
+
+ :param timeout:
+ Timeout in seconds
+
+ :param js_abort_controller:
+ A JavaScript AbortController object, used on timeout
+
+ :param request:
+ The request being handled
+
+ :param response:
+ The response being handled (if it exists yet)
+
+ :raises _TimeoutError: If the request times out
+ :raises _RequestError: If the request raises a JavaScript exception
+
+ :return: The result of awaiting the promise.
+ """
+ timer_id = None
+ if timeout > 0:
+ timer_id = js.setTimeout(
+ js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000)
+ )
+ try:
+ from pyodide.ffi import run_sync
+
+ # run_sync here uses WebAssembly JavaScript Promise Integration to
+ # suspend python until the JavaScript promise resolves.
+ return run_sync(promise)
+ except JsException as err:
+ if err.name == "AbortError":
+ raise _TimeoutError(
+ message="Request timed out", request=request, response=response
+ )
+ else:
+ raise _RequestError(message=err.message, request=request, response=response)
+ finally:
+ if timer_id is not None:
+ js.clearTimeout(timer_id)
+
+
+def has_jspi() -> bool:
+ """
+ Return true if jspi can be used.
+
+ This requires both browser support and also WebAssembly
+ to be in the correct state - i.e. that the javascript
+ call into python was async not sync.
+
+ :return: True if jspi can be used.
+ :rtype: bool
+ """
+ try:
+ from pyodide.ffi import can_run_sync, run_sync # noqa: F401
+
+ return bool(can_run_sync())
+ except ImportError:
+ return False
+
+
+def _is_node_js() -> bool:
+ """
+ Check if we are in Node.js.
+
+ :return: True if we are in Node.js.
+ :rtype: bool
+ """
+ return (
+ hasattr(js, "process")
+ and hasattr(js.process, "release")
+ # According to the Node.js documentation, the release name is always "node".
+ and js.process.release.name == "node"
+ )
+
+
+def streaming_ready() -> bool | None:
+ if _fetcher:
+ return _fetcher.streaming_ready
+ else:
+ return None # no fetcher, return None to signify that
+
+
+async def wait_for_streaming_ready() -> bool:
+ if _fetcher:
+ await _fetcher.js_worker_ready_promise
+ return True
+ else:
+ return False
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/request.py b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/request.py
new file mode 100644
index 0000000000000000000000000000000000000000..e692e692bd0d38f6a0677992a6993fc68050dff3
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/request.py
@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..._base_connection import _TYPE_BODY
+
+
+@dataclass
+class EmscriptenRequest:
+ method: str
+ url: str
+ params: dict[str, str] | None = None
+ body: _TYPE_BODY | None = None
+ headers: dict[str, str] = field(default_factory=dict)
+ timeout: float = 0
+ decode_content: bool = True
+
+ def set_header(self, name: str, value: str) -> None:
+ self.headers[name.capitalize()] = value
+
+ def set_body(self, body: _TYPE_BODY | None) -> None:
+ self.body = body
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/emscripten/response.py b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/response.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb1088a1826d089e1b603c51e85560b8583a3e3d
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/emscripten/response.py
@@ -0,0 +1,277 @@
+from __future__ import annotations
+
+import json as _json
+import logging
+import typing
+from contextlib import contextmanager
+from dataclasses import dataclass
+from http.client import HTTPException as HTTPException
+from io import BytesIO, IOBase
+
+from ...exceptions import InvalidHeader, TimeoutError
+from ...response import BaseHTTPResponse
+from ...util.retry import Retry
+from .request import EmscriptenRequest
+
+if typing.TYPE_CHECKING:
+ from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class EmscriptenResponse:
+ status_code: int
+ headers: dict[str, str]
+ body: IOBase | bytes
+ request: EmscriptenRequest
+
+
+class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
+ def __init__(
+ self,
+ internal_response: EmscriptenResponse,
+ url: str | None = None,
+ connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
+ ):
+ self._pool = None # set by pool class
+ self._body = None
+ self._response = internal_response
+ self._url = url
+ self._connection = connection
+ self._closed = False
+ super().__init__(
+ headers=internal_response.headers,
+ status=internal_response.status_code,
+ request_url=url,
+ version=0,
+ version_string="HTTP/?",
+ reason="",
+ decode_content=True,
+ )
+ self.length_remaining = self._init_length(self._response.request.method)
+ self.length_is_certain = False
+
+ @property
+ def url(self) -> str | None:
+ return self._url
+
+ @url.setter
+ def url(self, url: str | None) -> None:
+ self._url = url
+
+ @property
+ def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
+ return self._connection
+
+ @property
+ def retries(self) -> Retry | None:
+ return self._retries
+
+ @retries.setter
+ def retries(self, retries: Retry | None) -> None:
+ # Override the request_url if retries has a redirect location.
+ self._retries = retries
+
+ def stream(
+ self, amt: int | None = 2**16, decode_content: bool | None = None
+ ) -> typing.Generator[bytes]:
+ """
+ A generator wrapper for the read() method. A call will block until
+ ``amt`` bytes have been read from the connection or until the
+ connection is closed.
+
+ :param amt:
+ How much of the content to read. The generator will return up to
+ much data per iteration, but may return less. This is particularly
+ likely when using compressed data. However, the empty string will
+ never be returned.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ while True:
+ data = self.read(amt=amt, decode_content=decode_content)
+
+ if data:
+ yield data
+ else:
+ break
+
+ def _init_length(self, request_method: str | None) -> int | None:
+ length: int | None
+ content_length: str | None = self.headers.get("content-length")
+
+ if content_length is not None:
+ try:
+ # RFC 7230 section 3.3.2 specifies multiple content lengths can
+ # be sent in a single Content-Length header
+ # (e.g. Content-Length: 42, 42). This line ensures the values
+ # are all valid ints and that as long as the `set` length is 1,
+ # all values are the same. Otherwise, the header is invalid.
+ lengths = {int(val) for val in content_length.split(",")}
+ if len(lengths) > 1:
+ raise InvalidHeader(
+ "Content-Length contained multiple "
+ "unmatching values (%s)" % content_length
+ )
+ length = lengths.pop()
+ except ValueError:
+ length = None
+ else:
+ if length < 0:
+ length = None
+
+ else: # if content_length is None
+ length = None
+
+ # Check for responses that shouldn't include a body
+ if (
+ self.status in (204, 304)
+ or 100 <= self.status < 200
+ or request_method == "HEAD"
+ ):
+ length = 0
+
+ return length
+
+ def read(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None, # ignored because browser decodes always
+ cache_content: bool = False,
+ ) -> bytes:
+ if (
+ self._closed
+ or self._response is None
+ or (isinstance(self._response.body, IOBase) and self._response.body.closed)
+ ):
+ return b""
+
+ with self._error_catcher():
+ # body has been preloaded as a string by XmlHttpRequest
+ if not isinstance(self._response.body, IOBase):
+ self.length_remaining = len(self._response.body)
+ self.length_is_certain = True
+ # wrap body in IOStream
+ self._response.body = BytesIO(self._response.body)
+ if amt is not None and amt >= 0:
+ # don't cache partial content
+ cache_content = False
+ data = self._response.body.read(amt)
+ else: # read all we can (and cache it)
+ data = self._response.body.read()
+ if cache_content:
+ self._body = data
+ if self.length_remaining is not None:
+ self.length_remaining = max(self.length_remaining - len(data), 0)
+ if len(data) == 0 or (
+ self.length_is_certain and self.length_remaining == 0
+ ):
+ # definitely finished reading, close response stream
+ self._response.body.close()
+ return typing.cast(bytes, data)
+
+ def read_chunked(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ ) -> typing.Generator[bytes]:
+ # chunked is handled by browser
+ while True:
+ bytes = self.read(amt, decode_content)
+ if not bytes:
+ break
+ yield bytes
+
+ def release_conn(self) -> None:
+ if not self._pool or not self._connection:
+ return None
+
+ self._pool._put_conn(self._connection)
+ self._connection = None
+
+ def drain_conn(self) -> None:
+ self.close()
+
+ @property
+ def data(self) -> bytes:
+ if self._body:
+ return self._body
+ else:
+ return self.read(cache_content=True)
+
+ def json(self) -> typing.Any:
+ """
+ Deserializes the body of the HTTP response as a Python object.
+
+ The body of the HTTP response must be encoded using UTF-8, as per
+ `RFC 8529 Section 8.1 `_.
+
+ To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
+ your custom decoder instead.
+
+ If the body of the HTTP response is not decodable to UTF-8, a
+ `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
+ valid JSON document, a `json.JSONDecodeError` will be raised.
+
+ Read more :ref:`here `.
+
+ :returns: The body of the HTTP response as a Python object.
+ """
+ data = self.data.decode("utf-8")
+ return _json.loads(data)
+
+ def close(self) -> None:
+ if not self._closed:
+ if isinstance(self._response.body, IOBase):
+ self._response.body.close()
+ if self._connection:
+ self._connection.close()
+ self._connection = None
+ self._closed = True
+
+ @contextmanager
+ def _error_catcher(self) -> typing.Generator[None]:
+ """
+ Catch Emscripten specific exceptions thrown by fetch.py,
+ instead re-raising urllib3 variants, so that low-level exceptions
+ are not leaked in the high-level api.
+
+ On exit, release the connection back to the pool.
+ """
+ from .fetch import _RequestError, _TimeoutError # avoid circular import
+
+ clean_exit = False
+
+ try:
+ yield
+ # If no exception is thrown, we should avoid cleaning up
+ # unnecessarily.
+ clean_exit = True
+ except _TimeoutError as e:
+ raise TimeoutError(str(e))
+ except _RequestError as e:
+ raise HTTPException(str(e))
+ finally:
+ # If we didn't terminate cleanly, we need to throw away our
+ # connection.
+ if not clean_exit:
+ # The response may not be closed but we're not going to use it
+ # anymore so close it now
+ if (
+ isinstance(self._response.body, IOBase)
+ and not self._response.body.closed
+ ):
+ self._response.body.close()
+ # release the connection back to the pool
+ self.release_conn()
+ else:
+ # If we have read everything from the response stream,
+ # return the connection back to the pool.
+ if (
+ isinstance(self._response.body, IOBase)
+ and self._response.body.closed
+ ):
+ self.release_conn()
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/pyopenssl.py b/phivenv/Lib/site-packages/urllib3/contrib/pyopenssl.py
new file mode 100644
index 0000000000000000000000000000000000000000..3714500ec2e1c104b92dff1bb8f7244b8b458ed8
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/pyopenssl.py
@@ -0,0 +1,564 @@
+"""
+Module for using pyOpenSSL as a TLS backend. This module was relevant before
+the standard library ``ssl`` module supported SNI, but now that we've dropped
+support for Python 2.7 all relevant Python versions support SNI so
+**this module is no longer recommended**.
+
+This needs the following packages installed:
+
+* `pyOpenSSL`_ (tested with 16.0.0)
+* `cryptography`_ (minimum 1.3.4, from pyopenssl)
+* `idna`_ (minimum 2.0)
+
+However, pyOpenSSL depends on cryptography, so while we use all three directly here we
+end up having relatively few packages required.
+
+You can install them with the following command:
+
+.. code-block:: bash
+
+ $ python -m pip install pyopenssl cryptography idna
+
+To activate certificate checking, call
+:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
+before you begin making HTTP requests. This can be done in a ``sitecustomize``
+module, or at any other time before your application begins using ``urllib3``,
+like this:
+
+.. code-block:: python
+
+ try:
+ import urllib3.contrib.pyopenssl
+ urllib3.contrib.pyopenssl.inject_into_urllib3()
+ except ImportError:
+ pass
+
+.. _pyopenssl: https://www.pyopenssl.org
+.. _cryptography: https://cryptography.io
+.. _idna: https://github.com/kjd/idna
+"""
+
+from __future__ import annotations
+
+import OpenSSL.SSL # type: ignore[import-untyped]
+from cryptography import x509
+
+try:
+ from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
+except ImportError:
+ # UnsupportedExtension is gone in cryptography >= 2.1.0
+ class UnsupportedExtension(Exception): # type: ignore[no-redef]
+ pass
+
+
+import logging
+import ssl
+import typing
+from io import BytesIO
+from socket import socket as socket_cls
+from socket import timeout
+
+from .. import util
+
+if typing.TYPE_CHECKING:
+ from OpenSSL.crypto import X509 # type: ignore[import-untyped]
+
+
+__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
+
+# Map from urllib3 to PyOpenSSL compatible parameter-values.
+_openssl_versions: dict[int, int] = {
+ util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
+ util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
+ ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
+}
+
+if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
+
+if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
+ _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
+
+
+_stdlib_to_openssl_verify = {
+ ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
+ ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
+ ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+ + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
+}
+_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
+
+# The SSLvX values are the most likely to be missing in the future
+# but we check them all just to be sure.
+_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
+ OpenSSL.SSL, "OP_NO_SSLv3", 0
+)
+_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
+_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
+_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
+_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
+
+_openssl_to_ssl_minimum_version: dict[int, int] = {
+ ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
+ ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
+ ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
+ ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
+ ssl.TLSVersion.TLSv1_3: (
+ _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
+ ),
+ ssl.TLSVersion.MAXIMUM_SUPPORTED: (
+ _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
+ ),
+}
+_openssl_to_ssl_maximum_version: dict[int, int] = {
+ ssl.TLSVersion.MINIMUM_SUPPORTED: (
+ _OP_NO_SSLv2_OR_SSLv3
+ | _OP_NO_TLSv1
+ | _OP_NO_TLSv1_1
+ | _OP_NO_TLSv1_2
+ | _OP_NO_TLSv1_3
+ ),
+ ssl.TLSVersion.TLSv1: (
+ _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
+ ),
+ ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
+ ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
+ ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
+ ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
+}
+
+# OpenSSL will only write 16K at a time
+SSL_WRITE_BLOCKSIZE = 16384
+
+orig_util_SSLContext = util.ssl_.SSLContext
+
+
+log = logging.getLogger(__name__)
+
+
+def inject_into_urllib3() -> None:
+ "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
+
+ _validate_dependencies_met()
+
+ util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
+ util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
+ util.IS_PYOPENSSL = True
+ util.ssl_.IS_PYOPENSSL = True
+
+
+def extract_from_urllib3() -> None:
+ "Undo monkey-patching by :func:`inject_into_urllib3`."
+
+ util.SSLContext = orig_util_SSLContext
+ util.ssl_.SSLContext = orig_util_SSLContext
+ util.IS_PYOPENSSL = False
+ util.ssl_.IS_PYOPENSSL = False
+
+
+def _validate_dependencies_met() -> None:
+ """
+ Verifies that PyOpenSSL's package-level dependencies have been met.
+ Throws `ImportError` if they are not met.
+ """
+ # Method added in `cryptography==1.1`; not available in older versions
+ from cryptography.x509.extensions import Extensions
+
+ if getattr(Extensions, "get_extension_for_class", None) is None:
+ raise ImportError(
+ "'cryptography' module missing required functionality. "
+ "Try upgrading to v1.3.4 or newer."
+ )
+
+ # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
+ # attribute is only present on those versions.
+ from OpenSSL.crypto import X509
+
+ x509 = X509()
+ if getattr(x509, "_x509", None) is None:
+ raise ImportError(
+ "'pyOpenSSL' module missing required functionality. "
+ "Try upgrading to v0.14 or newer."
+ )
+
+
+def _dnsname_to_stdlib(name: str) -> str | None:
+ """
+ Converts a dNSName SubjectAlternativeName field to the form used by the
+ standard library on the given Python version.
+
+ Cryptography produces a dNSName as a unicode string that was idna-decoded
+ from ASCII bytes. We need to idna-encode that string to get it back, and
+ then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
+ uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
+
+ If the name cannot be idna-encoded then we return None signalling that
+ the name given should be skipped.
+ """
+
+ def idna_encode(name: str) -> bytes | None:
+ """
+ Borrowed wholesale from the Python Cryptography Project. It turns out
+ that we can't just safely call `idna.encode`: it can explode for
+ wildcard names. This avoids that problem.
+ """
+ import idna
+
+ try:
+ for prefix in ["*.", "."]:
+ if name.startswith(prefix):
+ name = name[len(prefix) :]
+ return prefix.encode("ascii") + idna.encode(name)
+ return idna.encode(name)
+ except idna.core.IDNAError:
+ return None
+
+ # Don't send IPv6 addresses through the IDNA encoder.
+ if ":" in name:
+ return name
+
+ encoded_name = idna_encode(name)
+ if encoded_name is None:
+ return None
+ return encoded_name.decode("utf-8")
+
+
+def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
+ """
+ Given an PyOpenSSL certificate, provides all the subject alternative names.
+ """
+ cert = peer_cert.to_cryptography()
+
+ # We want to find the SAN extension. Ask Cryptography to locate it (it's
+ # faster than looping in Python)
+ try:
+ ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
+ except x509.ExtensionNotFound:
+ # No such extension, return the empty list.
+ return []
+ except (
+ x509.DuplicateExtension,
+ UnsupportedExtension,
+ x509.UnsupportedGeneralNameType,
+ UnicodeError,
+ ) as e:
+ # A problem has been found with the quality of the certificate. Assume
+ # no SAN field is present.
+ log.warning(
+ "A problem was encountered with the certificate that prevented "
+ "urllib3 from finding the SubjectAlternativeName field. This can "
+ "affect certificate validation. The error was %s",
+ e,
+ )
+ return []
+
+ # We want to return dNSName and iPAddress fields. We need to cast the IPs
+ # back to strings because the match_hostname function wants them as
+ # strings.
+ # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
+ # decoded. This is pretty frustrating, but that's what the standard library
+ # does with certificates, and so we need to attempt to do the same.
+ # We also want to skip over names which cannot be idna encoded.
+ names = [
+ ("DNS", name)
+ for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
+ if name is not None
+ ]
+ names.extend(
+ ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
+ )
+
+ return names
+
+
+class WrappedSocket:
+ """API-compatibility wrapper for Python OpenSSL's Connection-class."""
+
+ def __init__(
+ self,
+ connection: OpenSSL.SSL.Connection,
+ socket: socket_cls,
+ suppress_ragged_eofs: bool = True,
+ ) -> None:
+ self.connection = connection
+ self.socket = socket
+ self.suppress_ragged_eofs = suppress_ragged_eofs
+ self._io_refs = 0
+ self._closed = False
+
+ def fileno(self) -> int:
+ return self.socket.fileno()
+
+ # Copy-pasted from Python 3.5 source code
+ def _decref_socketios(self) -> None:
+ if self._io_refs > 0:
+ self._io_refs -= 1
+ if self._closed:
+ self.close()
+
+ def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
+ try:
+ data = self.connection.recv(*args, **kwargs)
+ except OpenSSL.SSL.SysCallError as e:
+ if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
+ return b""
+ else:
+ raise OSError(e.args[0], str(e)) from e
+ except OpenSSL.SSL.ZeroReturnError:
+ if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+ return b""
+ else:
+ raise
+ except OpenSSL.SSL.WantReadError as e:
+ if not util.wait_for_read(self.socket, self.socket.gettimeout()):
+ raise timeout("The read operation timed out") from e
+ else:
+ return self.recv(*args, **kwargs)
+
+ # TLS 1.3 post-handshake authentication
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"read error: {e!r}") from e
+ else:
+ return data # type: ignore[no-any-return]
+
+ def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
+ try:
+ return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
+ except OpenSSL.SSL.SysCallError as e:
+ if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
+ return 0
+ else:
+ raise OSError(e.args[0], str(e)) from e
+ except OpenSSL.SSL.ZeroReturnError:
+ if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
+ return 0
+ else:
+ raise
+ except OpenSSL.SSL.WantReadError as e:
+ if not util.wait_for_read(self.socket, self.socket.gettimeout()):
+ raise timeout("The read operation timed out") from e
+ else:
+ return self.recv_into(*args, **kwargs)
+
+ # TLS 1.3 post-handshake authentication
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"read error: {e!r}") from e
+
+ def settimeout(self, timeout: float) -> None:
+ return self.socket.settimeout(timeout)
+
+ def _send_until_done(self, data: bytes) -> int:
+ while True:
+ try:
+ return self.connection.send(data) # type: ignore[no-any-return]
+ except OpenSSL.SSL.WantWriteError as e:
+ if not util.wait_for_write(self.socket, self.socket.gettimeout()):
+ raise timeout() from e
+ continue
+ except OpenSSL.SSL.SysCallError as e:
+ raise OSError(e.args[0], str(e)) from e
+
+ def sendall(self, data: bytes) -> None:
+ total_sent = 0
+ while total_sent < len(data):
+ sent = self._send_until_done(
+ data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
+ )
+ total_sent += sent
+
+ def shutdown(self, how: int) -> None:
+ try:
+ self.connection.shutdown()
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"shutdown error: {e!r}") from e
+
+ def close(self) -> None:
+ self._closed = True
+ if self._io_refs <= 0:
+ self._real_close()
+
+ def _real_close(self) -> None:
+ try:
+ return self.connection.close() # type: ignore[no-any-return]
+ except OpenSSL.SSL.Error:
+ return
+
+ def getpeercert(
+ self, binary_form: bool = False
+ ) -> dict[str, list[typing.Any]] | None:
+ x509 = self.connection.get_peer_certificate()
+
+ if not x509:
+ return x509 # type: ignore[no-any-return]
+
+ if binary_form:
+ return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
+
+ return {
+ "subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
+ "subjectAltName": get_subj_alt_name(x509),
+ }
+
+ def version(self) -> str:
+ return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
+
+ def selected_alpn_protocol(self) -> str | None:
+ alpn_proto = self.connection.get_alpn_proto_negotiated()
+ return alpn_proto.decode() if alpn_proto else None
+
+
+WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
+
+
+class PyOpenSSLContext:
+ """
+ I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
+ for translating the interface of the standard library ``SSLContext`` object
+ to calls into PyOpenSSL.
+ """
+
+ def __init__(self, protocol: int) -> None:
+ self.protocol = _openssl_versions[protocol]
+ self._ctx = OpenSSL.SSL.Context(self.protocol)
+ self._options = 0
+ self.check_hostname = False
+ self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
+ self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
+ self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST
+
+ @property
+ def options(self) -> int:
+ return self._options
+
+ @options.setter
+ def options(self, value: int) -> None:
+ self._options = value
+ self._set_ctx_options()
+
+ @property
+ def verify_flags(self) -> int:
+ return self._verify_flags
+
+ @verify_flags.setter
+ def verify_flags(self, value: int) -> None:
+ self._verify_flags = value
+ self._ctx.get_cert_store().set_flags(self._verify_flags)
+
+ @property
+ def verify_mode(self) -> int:
+ return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
+
+ @verify_mode.setter
+ def verify_mode(self, value: ssl.VerifyMode) -> None:
+ self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
+
+ def set_default_verify_paths(self) -> None:
+ self._ctx.set_default_verify_paths()
+
+ def set_ciphers(self, ciphers: bytes | str) -> None:
+ if isinstance(ciphers, str):
+ ciphers = ciphers.encode("utf-8")
+ self._ctx.set_cipher_list(ciphers)
+
+ def load_verify_locations(
+ self,
+ cafile: str | None = None,
+ capath: str | None = None,
+ cadata: bytes | None = None,
+ ) -> None:
+ if cafile is not None:
+ cafile = cafile.encode("utf-8") # type: ignore[assignment]
+ if capath is not None:
+ capath = capath.encode("utf-8") # type: ignore[assignment]
+ try:
+ self._ctx.load_verify_locations(cafile, capath)
+ if cadata is not None:
+ self._ctx.load_verify_locations(BytesIO(cadata))
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
+
+ def load_cert_chain(
+ self,
+ certfile: str,
+ keyfile: str | None = None,
+ password: str | None = None,
+ ) -> None:
+ try:
+ self._ctx.use_certificate_chain_file(certfile)
+ if password is not None:
+ if not isinstance(password, bytes):
+ password = password.encode("utf-8") # type: ignore[assignment]
+ self._ctx.set_passwd_cb(lambda *_: password)
+ self._ctx.use_privatekey_file(keyfile or certfile)
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
+
+ def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
+ protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
+ return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
+
+ def wrap_socket(
+ self,
+ sock: socket_cls,
+ server_side: bool = False,
+ do_handshake_on_connect: bool = True,
+ suppress_ragged_eofs: bool = True,
+ server_hostname: bytes | str | None = None,
+ ) -> WrappedSocket:
+ cnx = OpenSSL.SSL.Connection(self._ctx, sock)
+
+ # If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
+ if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
+ if isinstance(server_hostname, str):
+ server_hostname = server_hostname.encode("utf-8")
+ cnx.set_tlsext_host_name(server_hostname)
+
+ cnx.set_connect_state()
+
+ while True:
+ try:
+ cnx.do_handshake()
+ except OpenSSL.SSL.WantReadError as e:
+ if not util.wait_for_read(sock, sock.gettimeout()):
+ raise timeout("select timed out") from e
+ continue
+ except OpenSSL.SSL.Error as e:
+ raise ssl.SSLError(f"bad handshake: {e!r}") from e
+ break
+
+ return WrappedSocket(cnx, sock)
+
+ def _set_ctx_options(self) -> None:
+ self._ctx.set_options(
+ self._options
+ | _openssl_to_ssl_minimum_version[self._minimum_version]
+ | _openssl_to_ssl_maximum_version[self._maximum_version]
+ )
+
+ @property
+ def minimum_version(self) -> int:
+ return self._minimum_version
+
+ @minimum_version.setter
+ def minimum_version(self, minimum_version: int) -> None:
+ self._minimum_version = minimum_version
+ self._set_ctx_options()
+
+ @property
+ def maximum_version(self) -> int:
+ return self._maximum_version
+
+ @maximum_version.setter
+ def maximum_version(self, maximum_version: int) -> None:
+ self._maximum_version = maximum_version
+ self._set_ctx_options()
+
+
+def _verify_callback(
+ cnx: OpenSSL.SSL.Connection,
+ x509: X509,
+ err_no: int,
+ err_depth: int,
+ return_code: int,
+) -> bool:
+ return err_no == 0
diff --git a/phivenv/Lib/site-packages/urllib3/contrib/socks.py b/phivenv/Lib/site-packages/urllib3/contrib/socks.py
new file mode 100644
index 0000000000000000000000000000000000000000..c62b5e0332e7004d3a20da1fd555033f6f88780b
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/contrib/socks.py
@@ -0,0 +1,228 @@
+"""
+This module contains provisional support for SOCKS proxies from within
+urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
+SOCKS5. To enable its functionality, either install PySocks or install this
+module with the ``socks`` extra.
+
+The SOCKS implementation supports the full range of urllib3 features. It also
+supports the following SOCKS features:
+
+- SOCKS4A (``proxy_url='socks4a://...``)
+- SOCKS4 (``proxy_url='socks4://...``)
+- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
+- SOCKS5 with local DNS (``proxy_url='socks5://...``)
+- Usernames and passwords for the SOCKS proxy
+
+.. note::
+ It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
+ your ``proxy_url`` to ensure that DNS resolution is done from the remote
+ server instead of client-side when connecting to a domain name.
+
+SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
+supports IPv4, IPv6, and domain names.
+
+When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
+will be sent as the ``userid`` section of the SOCKS request:
+
+.. code-block:: python
+
+ proxy_url="socks4a://@proxy-host"
+
+When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
+of the ``proxy_url`` will be sent as the username/password to authenticate
+with the proxy:
+
+.. code-block:: python
+
+ proxy_url="socks5h://:@proxy-host"
+
+"""
+
+from __future__ import annotations
+
+try:
+ import socks # type: ignore[import-not-found]
+except ImportError:
+ import warnings
+
+ from ..exceptions import DependencyWarning
+
+ warnings.warn(
+ (
+ "SOCKS support in urllib3 requires the installation of optional "
+ "dependencies: specifically, PySocks. For more information, see "
+ "https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
+ ),
+ DependencyWarning,
+ )
+ raise
+
+import typing
+from socket import timeout as SocketTimeout
+
+from ..connection import HTTPConnection, HTTPSConnection
+from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from ..exceptions import ConnectTimeoutError, NewConnectionError
+from ..poolmanager import PoolManager
+from ..util.url import parse_url
+
+try:
+ import ssl
+except ImportError:
+ ssl = None # type: ignore[assignment]
+
+
+class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
+ socks_version: int
+ proxy_host: str | None
+ proxy_port: str | None
+ username: str | None
+ password: str | None
+ rdns: bool
+
+
+class SOCKSConnection(HTTPConnection):
+ """
+ A plain-text HTTP connection that connects via a SOCKS proxy.
+ """
+
+ def __init__(
+ self,
+ _socks_options: _TYPE_SOCKS_OPTIONS,
+ *args: typing.Any,
+ **kwargs: typing.Any,
+ ) -> None:
+ self._socks_options = _socks_options
+ super().__init__(*args, **kwargs)
+
+ def _new_conn(self) -> socks.socksocket:
+ """
+ Establish a new connection via the SOCKS proxy.
+ """
+ extra_kw: dict[str, typing.Any] = {}
+ if self.source_address:
+ extra_kw["source_address"] = self.source_address
+
+ if self.socket_options:
+ extra_kw["socket_options"] = self.socket_options
+
+ try:
+ conn = socks.create_connection(
+ (self.host, self.port),
+ proxy_type=self._socks_options["socks_version"],
+ proxy_addr=self._socks_options["proxy_host"],
+ proxy_port=self._socks_options["proxy_port"],
+ proxy_username=self._socks_options["username"],
+ proxy_password=self._socks_options["password"],
+ proxy_rdns=self._socks_options["rdns"],
+ timeout=self.timeout,
+ **extra_kw,
+ )
+
+ except SocketTimeout as e:
+ raise ConnectTimeoutError(
+ self,
+ f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
+ ) from e
+
+ except socks.ProxyError as e:
+ # This is fragile as hell, but it seems to be the only way to raise
+ # useful errors here.
+ if e.socket_err:
+ error = e.socket_err
+ if isinstance(error, SocketTimeout):
+ raise ConnectTimeoutError(
+ self,
+ f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
+ ) from e
+ else:
+ # Adding `from e` messes with coverage somehow, so it's omitted.
+ # See #2386.
+ raise NewConnectionError(
+ self, f"Failed to establish a new connection: {error}"
+ )
+ else:
+ raise NewConnectionError(
+ self, f"Failed to establish a new connection: {e}"
+ ) from e
+
+ except OSError as e: # Defensive: PySocks should catch all these.
+ raise NewConnectionError(
+ self, f"Failed to establish a new connection: {e}"
+ ) from e
+
+ return conn
+
+
+# We don't need to duplicate the Verified/Unverified distinction from
+# urllib3/connection.py here because the HTTPSConnection will already have been
+# correctly set to either the Verified or Unverified form by that module. This
+# means the SOCKSHTTPSConnection will automatically be the correct type.
+class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
+ pass
+
+
+class SOCKSHTTPConnectionPool(HTTPConnectionPool):
+ ConnectionCls = SOCKSConnection
+
+
+class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
+ ConnectionCls = SOCKSHTTPSConnection
+
+
+class SOCKSProxyManager(PoolManager):
+ """
+ A version of the urllib3 ProxyManager that routes connections via the
+ defined SOCKS proxy.
+ """
+
+ pool_classes_by_scheme = {
+ "http": SOCKSHTTPConnectionPool,
+ "https": SOCKSHTTPSConnectionPool,
+ }
+
+ def __init__(
+ self,
+ proxy_url: str,
+ username: str | None = None,
+ password: str | None = None,
+ num_pools: int = 10,
+ headers: typing.Mapping[str, str] | None = None,
+ **connection_pool_kw: typing.Any,
+ ):
+ parsed = parse_url(proxy_url)
+
+ if username is None and password is None and parsed.auth is not None:
+ split = parsed.auth.split(":")
+ if len(split) == 2:
+ username, password = split
+ if parsed.scheme == "socks5":
+ socks_version = socks.PROXY_TYPE_SOCKS5
+ rdns = False
+ elif parsed.scheme == "socks5h":
+ socks_version = socks.PROXY_TYPE_SOCKS5
+ rdns = True
+ elif parsed.scheme == "socks4":
+ socks_version = socks.PROXY_TYPE_SOCKS4
+ rdns = False
+ elif parsed.scheme == "socks4a":
+ socks_version = socks.PROXY_TYPE_SOCKS4
+ rdns = True
+ else:
+ raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
+
+ self.proxy_url = proxy_url
+
+ socks_options = {
+ "socks_version": socks_version,
+ "proxy_host": parsed.host,
+ "proxy_port": parsed.port,
+ "username": username,
+ "password": password,
+ "rdns": rdns,
+ }
+ connection_pool_kw["_socks_options"] = socks_options
+
+ super().__init__(num_pools, headers, **connection_pool_kw)
+
+ self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
diff --git a/phivenv/Lib/site-packages/urllib3/exceptions.py b/phivenv/Lib/site-packages/urllib3/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0de9d6cc4f0cc88123d696daa4fd095e1743ece
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/exceptions.py
@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+import socket
+import typing
+import warnings
+from email.errors import MessageDefect
+from http.client import IncompleteRead as httplib_IncompleteRead
+
+if typing.TYPE_CHECKING:
+ from .connection import HTTPConnection
+ from .connectionpool import ConnectionPool
+ from .response import HTTPResponse
+ from .util.retry import Retry
+
+# Base Exceptions
+
+
+class HTTPError(Exception):
+ """Base exception used by this module."""
+
+
+class HTTPWarning(Warning):
+ """Base warning used by this module."""
+
+
+_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]]
+
+
+class PoolError(HTTPError):
+ """Base exception for errors caused within a pool."""
+
+ def __init__(self, pool: ConnectionPool, message: str) -> None:
+ self.pool = pool
+ self._message = message
+ super().__init__(f"{pool}: {message}")
+
+ def __reduce__(self) -> _TYPE_REDUCE_RESULT:
+ # For pickling purposes.
+ return self.__class__, (None, self._message)
+
+
+class RequestError(PoolError):
+ """Base exception for PoolErrors that have associated URLs."""
+
+ def __init__(self, pool: ConnectionPool, url: str, message: str) -> None:
+ self.url = url
+ super().__init__(pool, message)
+
+ def __reduce__(self) -> _TYPE_REDUCE_RESULT:
+ # For pickling purposes.
+ return self.__class__, (None, self.url, self._message)
+
+
+class SSLError(HTTPError):
+ """Raised when SSL certificate fails in an HTTPS connection."""
+
+
+class ProxyError(HTTPError):
+ """Raised when the connection to a proxy fails."""
+
+ # The original error is also available as __cause__.
+ original_error: Exception
+
+ def __init__(self, message: str, error: Exception) -> None:
+ super().__init__(message, error)
+ self.original_error = error
+
+
+class DecodeError(HTTPError):
+ """Raised when automatic decoding based on Content-Type fails."""
+
+
+class ProtocolError(HTTPError):
+ """Raised when something unexpected happens mid-request/response."""
+
+
+#: Renamed to ProtocolError but aliased for backwards compatibility.
+ConnectionError = ProtocolError
+
+
+# Leaf Exceptions
+
+
+class MaxRetryError(RequestError):
+ """Raised when the maximum number of retries is exceeded.
+
+ :param pool: The connection pool
+ :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
+ :param str url: The requested Url
+ :param reason: The underlying error
+ :type reason: :class:`Exception`
+
+ """
+
+ def __init__(
+ self, pool: ConnectionPool, url: str, reason: Exception | None = None
+ ) -> None:
+ self.reason = reason
+
+ message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
+
+ super().__init__(pool, url, message)
+
+ def __reduce__(self) -> _TYPE_REDUCE_RESULT:
+ # For pickling purposes.
+ return self.__class__, (None, self.url, self.reason)
+
+
+class HostChangedError(RequestError):
+ """Raised when an existing pool gets a request for a foreign host."""
+
+ def __init__(
+ self, pool: ConnectionPool, url: str, retries: Retry | int = 3
+ ) -> None:
+ message = f"Tried to open a foreign host with url: {url}"
+ super().__init__(pool, url, message)
+ self.retries = retries
+
+
+class TimeoutStateError(HTTPError):
+ """Raised when passing an invalid state to a timeout"""
+
+
+class TimeoutError(HTTPError):
+ """Raised when a socket timeout error occurs.
+
+ Catching this error will catch both :exc:`ReadTimeoutErrors
+ ` and :exc:`ConnectTimeoutErrors `.
+ """
+
+
+class ReadTimeoutError(TimeoutError, RequestError):
+ """Raised when a socket timeout occurs while receiving data from a server"""
+
+
+# This timeout error does not have a URL attached and needs to inherit from the
+# base HTTPError
+class ConnectTimeoutError(TimeoutError):
+ """Raised when a socket timeout occurs while connecting to a server"""
+
+
+class NewConnectionError(ConnectTimeoutError, HTTPError):
+ """Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
+
+ def __init__(self, conn: HTTPConnection, message: str) -> None:
+ self.conn = conn
+ self._message = message
+ super().__init__(f"{conn}: {message}")
+
+ def __reduce__(self) -> _TYPE_REDUCE_RESULT:
+ # For pickling purposes.
+ return self.__class__, (None, self._message)
+
+ @property
+ def pool(self) -> HTTPConnection:
+ warnings.warn(
+ "The 'pool' property is deprecated and will be removed "
+ "in urllib3 v2.1.0. Use 'conn' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ return self.conn
+
+
+class NameResolutionError(NewConnectionError):
+ """Raised when host name resolution fails."""
+
+ def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
+ message = f"Failed to resolve '{host}' ({reason})"
+ self._host = host
+ self._reason = reason
+ super().__init__(conn, message)
+
+ def __reduce__(self) -> _TYPE_REDUCE_RESULT:
+ # For pickling purposes.
+ return self.__class__, (self._host, None, self._reason)
+
+
+class EmptyPoolError(PoolError):
+ """Raised when a pool runs out of connections and no more are allowed."""
+
+
+class FullPoolError(PoolError):
+ """Raised when we try to add a connection to a full pool in blocking mode."""
+
+
+class ClosedPoolError(PoolError):
+ """Raised when a request enters a pool after the pool has been closed."""
+
+
+class LocationValueError(ValueError, HTTPError):
+ """Raised when there is something wrong with a given URL input."""
+
+
+class LocationParseError(LocationValueError):
+ """Raised when get_host or similar fails to parse the URL input."""
+
+ def __init__(self, location: str) -> None:
+ message = f"Failed to parse: {location}"
+ super().__init__(message)
+
+ self.location = location
+
+
+class URLSchemeUnknown(LocationValueError):
+ """Raised when a URL input has an unsupported scheme."""
+
+ def __init__(self, scheme: str):
+ message = f"Not supported URL scheme {scheme}"
+ super().__init__(message)
+
+ self.scheme = scheme
+
+
+class ResponseError(HTTPError):
+ """Used as a container for an error reason supplied in a MaxRetryError."""
+
+ GENERIC_ERROR = "too many error responses"
+ SPECIFIC_ERROR = "too many {status_code} error responses"
+
+
+class SecurityWarning(HTTPWarning):
+ """Warned when performing security reducing actions"""
+
+
+class InsecureRequestWarning(SecurityWarning):
+ """Warned when making an unverified HTTPS request."""
+
+
+class NotOpenSSLWarning(SecurityWarning):
+ """Warned when using unsupported SSL library"""
+
+
+class SystemTimeWarning(SecurityWarning):
+ """Warned when system time is suspected to be wrong"""
+
+
+class InsecurePlatformWarning(SecurityWarning):
+ """Warned when certain TLS/SSL configuration is not available on a platform."""
+
+
+class DependencyWarning(HTTPWarning):
+ """
+ Warned when an attempt is made to import a module with missing optional
+ dependencies.
+ """
+
+
+class ResponseNotChunked(ProtocolError, ValueError):
+ """Response needs to be chunked in order to read it as chunks."""
+
+
+class BodyNotHttplibCompatible(HTTPError):
+ """
+ Body should be :class:`http.client.HTTPResponse` like
+ (have an fp attribute which returns raw chunks) for read_chunked().
+ """
+
+
+class IncompleteRead(HTTPError, httplib_IncompleteRead):
+ """
+ Response length doesn't match expected Content-Length
+
+ Subclass of :class:`http.client.IncompleteRead` to allow int value
+ for ``partial`` to avoid creating large objects on streamed reads.
+ """
+
+ partial: int # type: ignore[assignment]
+ expected: int
+
+ def __init__(self, partial: int, expected: int) -> None:
+ self.partial = partial
+ self.expected = expected
+
+ def __repr__(self) -> str:
+ return "IncompleteRead(%i bytes read, %i more expected)" % (
+ self.partial,
+ self.expected,
+ )
+
+
+class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
+ """Invalid chunk length in a chunked response."""
+
+ def __init__(self, response: HTTPResponse, length: bytes) -> None:
+ self.partial: int = response.tell() # type: ignore[assignment]
+ self.expected: int | None = response.length_remaining
+ self.response = response
+ self.length = length
+
+ def __repr__(self) -> str:
+ return "InvalidChunkLength(got length %r, %i bytes read)" % (
+ self.length,
+ self.partial,
+ )
+
+
+class InvalidHeader(HTTPError):
+ """The header provided was somehow invalid."""
+
+
+class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
+ """ProxyManager does not support the supplied scheme"""
+
+ # TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
+
+ def __init__(self, scheme: str | None) -> None:
+ # 'localhost' is here because our URL parser parses
+ # localhost:8080 -> scheme=localhost, remove if we fix this.
+ if scheme == "localhost":
+ scheme = None
+ if scheme is None:
+ message = "Proxy URL had no scheme, should start with http:// or https://"
+ else:
+ message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
+ super().__init__(message)
+
+
+class ProxySchemeUnsupported(ValueError):
+ """Fetching HTTPS resources through HTTPS proxies is unsupported"""
+
+
+class HeaderParsingError(HTTPError):
+ """Raised by assert_header_parsing, but we convert it to a log.warning statement."""
+
+ def __init__(
+ self, defects: list[MessageDefect], unparsed_data: bytes | str | None
+ ) -> None:
+ message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
+ super().__init__(message)
+
+
+class UnrewindableBodyError(HTTPError):
+ """urllib3 encountered an error when trying to rewind a body"""
diff --git a/phivenv/Lib/site-packages/urllib3/fields.py b/phivenv/Lib/site-packages/urllib3/fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..97c4730cff0df570e1ab47f77e6aa879ec3c36e7
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/fields.py
@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import email.utils
+import mimetypes
+import typing
+
+_TYPE_FIELD_VALUE = typing.Union[str, bytes]
+_TYPE_FIELD_VALUE_TUPLE = typing.Union[
+ _TYPE_FIELD_VALUE,
+ tuple[str, _TYPE_FIELD_VALUE],
+ tuple[str, _TYPE_FIELD_VALUE, str],
+]
+
+
+def guess_content_type(
+ filename: str | None, default: str = "application/octet-stream"
+) -> str:
+ """
+ Guess the "Content-Type" of a file.
+
+ :param filename:
+ The filename to guess the "Content-Type" of using :mod:`mimetypes`.
+ :param default:
+ If no "Content-Type" can be guessed, default to `default`.
+ """
+ if filename:
+ return mimetypes.guess_type(filename)[0] or default
+ return default
+
+
+def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
+ """
+ Helper function to format and quote a single header parameter using the
+ strategy defined in RFC 2231.
+
+ Particularly useful for header parameters which might contain
+ non-ASCII values, like file names. This follows
+ `RFC 2388 Section 4.4 `_.
+
+ :param name:
+ The name of the parameter, a string expected to be ASCII only.
+ :param value:
+ The value of the parameter, provided as ``bytes`` or `str``.
+ :returns:
+ An RFC-2231-formatted unicode string.
+
+ .. deprecated:: 2.0.0
+ Will be removed in urllib3 v2.1.0. This is not valid for
+ ``multipart/form-data`` header parameters.
+ """
+ import warnings
+
+ warnings.warn(
+ "'format_header_param_rfc2231' is deprecated and will be "
+ "removed in urllib3 v2.1.0. This is not valid for "
+ "multipart/form-data header parameters.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ if isinstance(value, bytes):
+ value = value.decode("utf-8")
+
+ if not any(ch in value for ch in '"\\\r\n'):
+ result = f'{name}="{value}"'
+ try:
+ result.encode("ascii")
+ except (UnicodeEncodeError, UnicodeDecodeError):
+ pass
+ else:
+ return result
+
+ value = email.utils.encode_rfc2231(value, "utf-8")
+ value = f"{name}*={value}"
+
+ return value
+
+
+def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
+ """
+ Format and quote a single multipart header parameter.
+
+ This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
+ the behavior of current browser and curl versions. Values are
+ assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
+ percent encoded.
+
+ .. _WHATWG HTML Standard:
+ https://html.spec.whatwg.org/multipage/
+ form-control-infrastructure.html#multipart-form-data
+
+ :param name:
+ The name of the parameter, an ASCII-only ``str``.
+ :param value:
+ The value of the parameter, a ``str`` or UTF-8 encoded
+ ``bytes``.
+ :returns:
+ A string ``name="value"`` with the escaped value.
+
+ .. versionchanged:: 2.0.0
+ Matches the WHATWG HTML Standard as of 2021/06/10. Control
+ characters are no longer percent encoded.
+
+ .. versionchanged:: 2.0.0
+ Renamed from ``format_header_param_html5`` and
+ ``format_header_param``. The old names will be removed in
+ urllib3 v2.1.0.
+ """
+ if isinstance(value, bytes):
+ value = value.decode("utf-8")
+
+ # percent encode \n \r "
+ value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
+ return f'{name}="{value}"'
+
+
+def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
+ """
+ .. deprecated:: 2.0.0
+ Renamed to :func:`format_multipart_header_param`. Will be
+ removed in urllib3 v2.1.0.
+ """
+ import warnings
+
+ warnings.warn(
+ "'format_header_param_html5' has been renamed to "
+ "'format_multipart_header_param'. The old name will be "
+ "removed in urllib3 v2.1.0.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return format_multipart_header_param(name, value)
+
+
+def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
+ """
+ .. deprecated:: 2.0.0
+ Renamed to :func:`format_multipart_header_param`. Will be
+ removed in urllib3 v2.1.0.
+ """
+ import warnings
+
+ warnings.warn(
+ "'format_header_param' has been renamed to "
+ "'format_multipart_header_param'. The old name will be "
+ "removed in urllib3 v2.1.0.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return format_multipart_header_param(name, value)
+
+
+class RequestField:
+ """
+ A data container for request body parameters.
+
+ :param name:
+ The name of this request field. Must be unicode.
+ :param data:
+ The data/value body.
+ :param filename:
+ An optional filename of the request field. Must be unicode.
+ :param headers:
+ An optional dict-like object of headers to initially use for the field.
+
+ .. versionchanged:: 2.0.0
+ The ``header_formatter`` parameter is deprecated and will
+ be removed in urllib3 v2.1.0.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ data: _TYPE_FIELD_VALUE,
+ filename: str | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
+ ):
+ self._name = name
+ self._filename = filename
+ self.data = data
+ self.headers: dict[str, str | None] = {}
+ if headers:
+ self.headers = dict(headers)
+
+ if header_formatter is not None:
+ import warnings
+
+ warnings.warn(
+ "The 'header_formatter' parameter is deprecated and "
+ "will be removed in urllib3 v2.1.0.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ self.header_formatter = header_formatter
+ else:
+ self.header_formatter = format_multipart_header_param
+
+ @classmethod
+ def from_tuples(
+ cls,
+ fieldname: str,
+ value: _TYPE_FIELD_VALUE_TUPLE,
+ header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
+ ) -> RequestField:
+ """
+ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
+
+ Supports constructing :class:`~urllib3.fields.RequestField` from
+ parameter of key/value strings AND key/filetuple. A filetuple is a
+ (filename, data, MIME type) tuple where the MIME type is optional.
+ For example::
+
+ 'foo': 'bar',
+ 'fakefile': ('foofile.txt', 'contents of foofile'),
+ 'realfile': ('barfile.txt', open('realfile').read()),
+ 'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
+ 'nonamefile': 'contents of nonamefile field',
+
+ Field names and filenames must be unicode.
+ """
+ filename: str | None
+ content_type: str | None
+ data: _TYPE_FIELD_VALUE
+
+ if isinstance(value, tuple):
+ if len(value) == 3:
+ filename, data, content_type = value
+ else:
+ filename, data = value
+ content_type = guess_content_type(filename)
+ else:
+ filename = None
+ content_type = None
+ data = value
+
+ request_param = cls(
+ fieldname, data, filename=filename, header_formatter=header_formatter
+ )
+ request_param.make_multipart(content_type=content_type)
+
+ return request_param
+
+ def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
+ """
+ Override this method to change how each multipart header
+ parameter is formatted. By default, this calls
+ :func:`format_multipart_header_param`.
+
+ :param name:
+ The name of the parameter, an ASCII-only ``str``.
+ :param value:
+ The value of the parameter, a ``str`` or UTF-8 encoded
+ ``bytes``.
+
+ :meta public:
+ """
+ return self.header_formatter(name, value)
+
+ def _render_parts(
+ self,
+ header_parts: (
+ dict[str, _TYPE_FIELD_VALUE | None]
+ | typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
+ ),
+ ) -> str:
+ """
+ Helper function to format and quote a single header.
+
+ Useful for single headers that are composed of multiple items. E.g.,
+ 'Content-Disposition' fields.
+
+ :param header_parts:
+ A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
+ as `k1="v1"; k2="v2"; ...`.
+ """
+ iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
+
+ parts = []
+ if isinstance(header_parts, dict):
+ iterable = header_parts.items()
+ else:
+ iterable = header_parts
+
+ for name, value in iterable:
+ if value is not None:
+ parts.append(self._render_part(name, value))
+
+ return "; ".join(parts)
+
+ def render_headers(self) -> str:
+ """
+ Renders the headers for this request field.
+ """
+ lines = []
+
+ sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
+ for sort_key in sort_keys:
+ if self.headers.get(sort_key, False):
+ lines.append(f"{sort_key}: {self.headers[sort_key]}")
+
+ for header_name, header_value in self.headers.items():
+ if header_name not in sort_keys:
+ if header_value:
+ lines.append(f"{header_name}: {header_value}")
+
+ lines.append("\r\n")
+ return "\r\n".join(lines)
+
+ def make_multipart(
+ self,
+ content_disposition: str | None = None,
+ content_type: str | None = None,
+ content_location: str | None = None,
+ ) -> None:
+ """
+ Makes this request field into a multipart request field.
+
+ This method overrides "Content-Disposition", "Content-Type" and
+ "Content-Location" headers to the request parameter.
+
+ :param content_disposition:
+ The 'Content-Disposition' of the request body. Defaults to 'form-data'
+ :param content_type:
+ The 'Content-Type' of the request body.
+ :param content_location:
+ The 'Content-Location' of the request body.
+
+ """
+ content_disposition = (content_disposition or "form-data") + "; ".join(
+ [
+ "",
+ self._render_parts(
+ (("name", self._name), ("filename", self._filename))
+ ),
+ ]
+ )
+
+ self.headers["Content-Disposition"] = content_disposition
+ self.headers["Content-Type"] = content_type
+ self.headers["Content-Location"] = content_location
diff --git a/phivenv/Lib/site-packages/urllib3/filepost.py b/phivenv/Lib/site-packages/urllib3/filepost.py
new file mode 100644
index 0000000000000000000000000000000000000000..14f70b05b4778f91137e4a9e7059d7514aa44d28
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/filepost.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import binascii
+import codecs
+import os
+import typing
+from io import BytesIO
+
+from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
+
+writer = codecs.lookup("utf-8")[3]
+
+_TYPE_FIELDS_SEQUENCE = typing.Sequence[
+ typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
+]
+_TYPE_FIELDS = typing.Union[
+ _TYPE_FIELDS_SEQUENCE,
+ typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
+]
+
+
+def choose_boundary() -> str:
+ """
+ Our embarrassingly-simple replacement for mimetools.choose_boundary.
+ """
+ return binascii.hexlify(os.urandom(16)).decode()
+
+
+def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
+ """
+ Iterate over fields.
+
+ Supports list of (k, v) tuples and dicts, and lists of
+ :class:`~urllib3.fields.RequestField`.
+
+ """
+ iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
+
+ if isinstance(fields, typing.Mapping):
+ iterable = fields.items()
+ else:
+ iterable = fields
+
+ for field in iterable:
+ if isinstance(field, RequestField):
+ yield field
+ else:
+ yield RequestField.from_tuples(*field)
+
+
+def encode_multipart_formdata(
+ fields: _TYPE_FIELDS, boundary: str | None = None
+) -> tuple[bytes, str]:
+ """
+ Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
+
+ :param fields:
+ Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
+ Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
+
+ :param boundary:
+ If not specified, then a random boundary will be generated using
+ :func:`urllib3.filepost.choose_boundary`.
+ """
+ body = BytesIO()
+ if boundary is None:
+ boundary = choose_boundary()
+
+ for field in iter_field_objects(fields):
+ body.write(f"--{boundary}\r\n".encode("latin-1"))
+
+ writer(body).write(field.render_headers())
+ data = field.data
+
+ if isinstance(data, int):
+ data = str(data) # Backwards compatibility
+
+ if isinstance(data, str):
+ writer(body).write(data)
+ else:
+ body.write(data)
+
+ body.write(b"\r\n")
+
+ body.write(f"--{boundary}--\r\n".encode("latin-1"))
+
+ content_type = f"multipart/form-data; boundary={boundary}"
+
+ return body.getvalue(), content_type
diff --git a/phivenv/Lib/site-packages/urllib3/http2/__init__.py b/phivenv/Lib/site-packages/urllib3/http2/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..133e1d8f237f6fddd557ae1c0e0cf738f7cc2748
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/http2/__init__.py
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from importlib.metadata import version
+
+__all__ = [
+ "inject_into_urllib3",
+ "extract_from_urllib3",
+]
+
+import typing
+
+orig_HTTPSConnection: typing.Any = None
+
+
+def inject_into_urllib3() -> None:
+ # First check if h2 version is valid
+ h2_version = version("h2")
+ if not h2_version.startswith("4."):
+ raise ImportError(
+ "urllib3 v2 supports h2 version 4.x.x, currently "
+ f"the 'h2' module is compiled with {h2_version!r}. "
+ "See: https://github.com/urllib3/urllib3/issues/3290"
+ )
+
+ # Import here to avoid circular dependencies.
+ from .. import connection as urllib3_connection
+ from .. import util as urllib3_util
+ from ..connectionpool import HTTPSConnectionPool
+ from ..util import ssl_ as urllib3_util_ssl
+ from .connection import HTTP2Connection
+
+ global orig_HTTPSConnection
+ orig_HTTPSConnection = urllib3_connection.HTTPSConnection
+
+ HTTPSConnectionPool.ConnectionCls = HTTP2Connection
+ urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc]
+
+ # TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
+ urllib3_util.ALPN_PROTOCOLS = ["h2"]
+ urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"]
+
+
+def extract_from_urllib3() -> None:
+ from .. import connection as urllib3_connection
+ from .. import util as urllib3_util
+ from ..connectionpool import HTTPSConnectionPool
+ from ..util import ssl_ as urllib3_util_ssl
+
+ HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
+ urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc]
+
+ urllib3_util.ALPN_PROTOCOLS = ["http/1.1"]
+ urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"]
diff --git a/phivenv/Lib/site-packages/urllib3/http2/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cd55da4f199dc374317709b1f8370afc62555c54
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/http2/__pycache__/connection.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/connection.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4bebbe7d2b8e1284339f90c191e476c12bbdaec6
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/connection.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/http2/__pycache__/probe.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/probe.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8ca2ac01258f6b75026355a5e39429d43172b6aa
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/http2/__pycache__/probe.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/http2/connection.py b/phivenv/Lib/site-packages/urllib3/http2/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0822391f6f6bd1c1203fd0c212100e7144c3553
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/http2/connection.py
@@ -0,0 +1,356 @@
+from __future__ import annotations
+
+import logging
+import re
+import threading
+import types
+import typing
+
+import h2.config # type: ignore[import-untyped]
+import h2.connection # type: ignore[import-untyped]
+import h2.events # type: ignore[import-untyped]
+
+from .._base_connection import _TYPE_BODY
+from .._collections import HTTPHeaderDict
+from ..connection import HTTPSConnection, _get_default_user_agent
+from ..exceptions import ConnectionError
+from ..response import BaseHTTPResponse
+
+orig_HTTPSConnection = HTTPSConnection
+
+T = typing.TypeVar("T")
+
+log = logging.getLogger(__name__)
+
+RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$")
+RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$")
+
+
+def _is_legal_header_name(name: bytes) -> bool:
+ """
+ "An implementation that validates fields according to the definitions in Sections
+ 5.1 and 5.5 of [HTTP] only needs an additional check that field names do not
+ include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
+
+ `http.client._is_legal_header_name` does not validate the field name according to the
+ HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters.
+
+ This does not allow for the `:` character in the header name, so should not
+ be used to validate pseudo-headers.
+ """
+ return bool(RE_IS_LEGAL_HEADER_NAME.match(name))
+
+
+def _is_illegal_header_value(value: bytes) -> bool:
+ """
+ "A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed
+ (ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field
+ value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB,
+ 0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
+ """
+ return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value))
+
+
+class _LockedObject(typing.Generic[T]):
+ """
+ A wrapper class that hides a specific object behind a lock.
+ The goal here is to provide a simple way to protect access to an object
+ that cannot safely be simultaneously accessed from multiple threads. The
+ intended use of this class is simple: take hold of it with a context
+ manager, which returns the protected object.
+ """
+
+ __slots__ = (
+ "lock",
+ "_obj",
+ )
+
+ def __init__(self, obj: T):
+ self.lock = threading.RLock()
+ self._obj = obj
+
+ def __enter__(self) -> T:
+ self.lock.acquire()
+ return self._obj
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: types.TracebackType | None,
+ ) -> None:
+ self.lock.release()
+
+
+class HTTP2Connection(HTTPSConnection):
+ def __init__(
+ self, host: str, port: int | None = None, **kwargs: typing.Any
+ ) -> None:
+ self._h2_conn = self._new_h2_conn()
+ self._h2_stream: int | None = None
+ self._headers: list[tuple[bytes, bytes]] = []
+
+ if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive:
+ raise NotImplementedError("Proxies aren't supported with HTTP/2")
+
+ super().__init__(host, port, **kwargs)
+
+ if self._tunnel_host is not None:
+ raise NotImplementedError("Tunneling isn't supported with HTTP/2")
+
+ def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]:
+ config = h2.config.H2Configuration(client_side=True)
+ return _LockedObject(h2.connection.H2Connection(config=config))
+
+ def connect(self) -> None:
+ super().connect()
+ with self._h2_conn as conn:
+ conn.initiate_connection()
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+
+ def putrequest( # type: ignore[override]
+ self,
+ method: str,
+ url: str,
+ **kwargs: typing.Any,
+ ) -> None:
+ """putrequest
+ This deviates from the HTTPConnection method signature since we never need to override
+ sending accept-encoding headers or the host header.
+ """
+ if "skip_host" in kwargs:
+ raise NotImplementedError("`skip_host` isn't supported")
+ if "skip_accept_encoding" in kwargs:
+ raise NotImplementedError("`skip_accept_encoding` isn't supported")
+
+ self._request_url = url or "/"
+ self._validate_path(url) # type: ignore[attr-defined]
+
+ if ":" in self.host:
+ authority = f"[{self.host}]:{self.port or 443}"
+ else:
+ authority = f"{self.host}:{self.port or 443}"
+
+ self._headers.append((b":scheme", b"https"))
+ self._headers.append((b":method", method.encode()))
+ self._headers.append((b":authority", authority.encode()))
+ self._headers.append((b":path", url.encode()))
+
+ with self._h2_conn as conn:
+ self._h2_stream = conn.get_next_available_stream_id()
+
+ def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override]
+ # TODO SKIPPABLE_HEADERS from urllib3 are ignored.
+ header = header.encode() if isinstance(header, str) else header
+ header = header.lower() # A lot of upstream code uses capitalized headers.
+ if not _is_legal_header_name(header):
+ raise ValueError(f"Illegal header name {str(header)}")
+
+ for value in values:
+ value = value.encode() if isinstance(value, str) else value
+ if _is_illegal_header_value(value):
+ raise ValueError(f"Illegal header value {str(value)}")
+ self._headers.append((header, value))
+
+ def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override]
+ if self._h2_stream is None:
+ raise ConnectionError("Must call `putrequest` first.")
+
+ with self._h2_conn as conn:
+ conn.send_headers(
+ stream_id=self._h2_stream,
+ headers=self._headers,
+ end_stream=(message_body is None),
+ )
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+ self._headers = [] # Reset headers for the next request.
+
+ def send(self, data: typing.Any) -> None:
+ """Send data to the server.
+ `data` can be: `str`, `bytes`, an iterable, or file-like objects
+ that support a .read() method.
+ """
+ if self._h2_stream is None:
+ raise ConnectionError("Must call `putrequest` first.")
+
+ with self._h2_conn as conn:
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+
+ if hasattr(data, "read"): # file-like objects
+ while True:
+ chunk = data.read(self.blocksize)
+ if not chunk:
+ break
+ if isinstance(chunk, str):
+ chunk = chunk.encode() # pragma: no cover
+ conn.send_data(self._h2_stream, chunk, end_stream=False)
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+ conn.end_stream(self._h2_stream)
+ return
+
+ if isinstance(data, str): # str -> bytes
+ data = data.encode()
+
+ try:
+ if isinstance(data, bytes):
+ conn.send_data(self._h2_stream, data, end_stream=True)
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+ else:
+ for chunk in data:
+ conn.send_data(self._h2_stream, chunk, end_stream=False)
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+ conn.end_stream(self._h2_stream)
+ except TypeError:
+ raise TypeError(
+ "`data` should be str, bytes, iterable, or file. got %r"
+ % type(data)
+ )
+
+ def set_tunnel(
+ self,
+ host: str,
+ port: int | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ scheme: str = "http",
+ ) -> None:
+ raise NotImplementedError(
+ "HTTP/2 does not support setting up a tunnel through a proxy"
+ )
+
+ def getresponse( # type: ignore[override]
+ self,
+ ) -> HTTP2Response:
+ status = None
+ data = bytearray()
+ with self._h2_conn as conn:
+ end_stream = False
+ while not end_stream:
+ # TODO: Arbitrary read value.
+ if received_data := self.sock.recv(65535):
+ events = conn.receive_data(received_data)
+ for event in events:
+ if isinstance(event, h2.events.ResponseReceived):
+ headers = HTTPHeaderDict()
+ for header, value in event.headers:
+ if header == b":status":
+ status = int(value.decode())
+ else:
+ headers.add(
+ header.decode("ascii"), value.decode("ascii")
+ )
+
+ elif isinstance(event, h2.events.DataReceived):
+ data += event.data
+ conn.acknowledge_received_data(
+ event.flow_controlled_length, event.stream_id
+ )
+
+ elif isinstance(event, h2.events.StreamEnded):
+ end_stream = True
+
+ if data_to_send := conn.data_to_send():
+ self.sock.sendall(data_to_send)
+
+ assert status is not None
+ return HTTP2Response(
+ status=status,
+ headers=headers,
+ request_url=self._request_url,
+ data=bytes(data),
+ )
+
+ def request( # type: ignore[override]
+ self,
+ method: str,
+ url: str,
+ body: _TYPE_BODY | None = None,
+ headers: typing.Mapping[str, str] | None = None,
+ *,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ enforce_content_length: bool = True,
+ **kwargs: typing.Any,
+ ) -> None:
+ """Send an HTTP/2 request"""
+ if "chunked" in kwargs:
+ # TODO this is often present from upstream.
+ # raise NotImplementedError("`chunked` isn't supported with HTTP/2")
+ pass
+
+ if self.sock is not None:
+ self.sock.settimeout(self.timeout)
+
+ self.putrequest(method, url)
+
+ headers = headers or {}
+ for k, v in headers.items():
+ if k.lower() == "transfer-encoding" and v == "chunked":
+ continue
+ else:
+ self.putheader(k, v)
+
+ if b"user-agent" not in dict(self._headers):
+ self.putheader(b"user-agent", _get_default_user_agent())
+
+ if body:
+ self.endheaders(message_body=body)
+ self.send(body)
+ else:
+ self.endheaders()
+
+ def close(self) -> None:
+ with self._h2_conn as conn:
+ try:
+ conn.close_connection()
+ if data := conn.data_to_send():
+ self.sock.sendall(data)
+ except Exception:
+ pass
+
+ # Reset all our HTTP/2 connection state.
+ self._h2_conn = self._new_h2_conn()
+ self._h2_stream = None
+ self._headers = []
+
+ super().close()
+
+
+class HTTP2Response(BaseHTTPResponse):
+ # TODO: This is a woefully incomplete response object, but works for non-streaming.
+ def __init__(
+ self,
+ status: int,
+ headers: HTTPHeaderDict,
+ request_url: str,
+ data: bytes,
+ decode_content: bool = False, # TODO: support decoding
+ ) -> None:
+ super().__init__(
+ status=status,
+ headers=headers,
+ # Following CPython, we map HTTP versions to major * 10 + minor integers
+ version=20,
+ version_string="HTTP/2",
+ # No reason phrase in HTTP/2
+ reason=None,
+ decode_content=decode_content,
+ request_url=request_url,
+ )
+ self._data = data
+ self.length_remaining = 0
+
+ @property
+ def data(self) -> bytes:
+ return self._data
+
+ def get_redirect_location(self) -> None:
+ return None
+
+ def close(self) -> None:
+ pass
diff --git a/phivenv/Lib/site-packages/urllib3/http2/probe.py b/phivenv/Lib/site-packages/urllib3/http2/probe.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ea900764f0885eafaac9454523417d86e33df2d
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/http2/probe.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import threading
+
+
+class _HTTP2ProbeCache:
+ __slots__ = (
+ "_lock",
+ "_cache_locks",
+ "_cache_values",
+ )
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._cache_locks: dict[tuple[str, int], threading.RLock] = {}
+ self._cache_values: dict[tuple[str, int], bool | None] = {}
+
+ def acquire_and_get(self, host: str, port: int) -> bool | None:
+ # By the end of this block we know that
+ # _cache_[values,locks] is available.
+ value = None
+ with self._lock:
+ key = (host, port)
+ try:
+ value = self._cache_values[key]
+ # If it's a known value we return right away.
+ if value is not None:
+ return value
+ except KeyError:
+ self._cache_locks[key] = threading.RLock()
+ self._cache_values[key] = None
+
+ # If the value is unknown, we acquire the lock to signal
+ # to the requesting thread that the probe is in progress
+ # or that the current thread needs to return their findings.
+ key_lock = self._cache_locks[key]
+ key_lock.acquire()
+ try:
+ # If the by the time we get the lock the value has been
+ # updated we want to return the updated value.
+ value = self._cache_values[key]
+
+ # In case an exception like KeyboardInterrupt is raised here.
+ except BaseException as e: # Defensive:
+ assert not isinstance(e, KeyError) # KeyError shouldn't be possible.
+ key_lock.release()
+ raise
+
+ return value
+
+ def set_and_release(
+ self, host: str, port: int, supports_http2: bool | None
+ ) -> None:
+ key = (host, port)
+ key_lock = self._cache_locks[key]
+ with key_lock: # Uses an RLock, so can be locked again from same thread.
+ if supports_http2 is None and self._cache_values[key] is not None:
+ raise ValueError(
+ "Cannot reset HTTP/2 support for origin after value has been set."
+ ) # Defensive: not expected in normal usage
+
+ self._cache_values[key] = supports_http2
+ key_lock.release()
+
+ def _values(self) -> dict[tuple[str, int], bool | None]:
+ """This function is for testing purposes only. Gets the current state of the probe cache"""
+ with self._lock:
+ return {k: v for k, v in self._cache_values.items()}
+
+ def _reset(self) -> None:
+ """This function is for testing purposes only. Reset the cache values"""
+ with self._lock:
+ self._cache_locks = {}
+ self._cache_values = {}
+
+
+_HTTP2_PROBE_CACHE = _HTTP2ProbeCache()
+
+set_and_release = _HTTP2_PROBE_CACHE.set_and_release
+acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get
+_values = _HTTP2_PROBE_CACHE._values
+_reset = _HTTP2_PROBE_CACHE._reset
+
+__all__ = [
+ "set_and_release",
+ "acquire_and_get",
+]
diff --git a/phivenv/Lib/site-packages/urllib3/poolmanager.py b/phivenv/Lib/site-packages/urllib3/poolmanager.py
new file mode 100644
index 0000000000000000000000000000000000000000..5763fea808184690042cca497b3ac358f412c83f
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/poolmanager.py
@@ -0,0 +1,653 @@
+from __future__ import annotations
+
+import functools
+import logging
+import typing
+import warnings
+from types import TracebackType
+from urllib.parse import urljoin
+
+from ._collections import HTTPHeaderDict, RecentlyUsedContainer
+from ._request_methods import RequestMethods
+from .connection import ProxyConfig
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
+from .exceptions import (
+ LocationValueError,
+ MaxRetryError,
+ ProxySchemeUnknown,
+ URLSchemeUnknown,
+)
+from .response import BaseHTTPResponse
+from .util.connection import _TYPE_SOCKET_OPTIONS
+from .util.proxy import connection_requires_http_tunnel
+from .util.retry import Retry
+from .util.timeout import Timeout
+from .util.url import Url, parse_url
+
+if typing.TYPE_CHECKING:
+ import ssl
+
+ from typing_extensions import Self
+
+__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
+
+
+log = logging.getLogger(__name__)
+
+SSL_KEYWORDS = (
+ "key_file",
+ "cert_file",
+ "cert_reqs",
+ "ca_certs",
+ "ca_cert_data",
+ "ssl_version",
+ "ssl_minimum_version",
+ "ssl_maximum_version",
+ "ca_cert_dir",
+ "ssl_context",
+ "key_password",
+ "server_hostname",
+)
+# Default value for `blocksize` - a new parameter introduced to
+# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
+_DEFAULT_BLOCKSIZE = 16384
+
+
+class PoolKey(typing.NamedTuple):
+ """
+ All known keyword arguments that could be provided to the pool manager, its
+ pools, or the underlying connections.
+
+ All custom key schemes should include the fields in this key at a minimum.
+ """
+
+ key_scheme: str
+ key_host: str
+ key_port: int | None
+ key_timeout: Timeout | float | int | None
+ key_retries: Retry | bool | int | None
+ key_block: bool | None
+ key_source_address: tuple[str, int] | None
+ key_key_file: str | None
+ key_key_password: str | None
+ key_cert_file: str | None
+ key_cert_reqs: str | None
+ key_ca_certs: str | None
+ key_ca_cert_data: str | bytes | None
+ key_ssl_version: int | str | None
+ key_ssl_minimum_version: ssl.TLSVersion | None
+ key_ssl_maximum_version: ssl.TLSVersion | None
+ key_ca_cert_dir: str | None
+ key_ssl_context: ssl.SSLContext | None
+ key_maxsize: int | None
+ key_headers: frozenset[tuple[str, str]] | None
+ key__proxy: Url | None
+ key__proxy_headers: frozenset[tuple[str, str]] | None
+ key__proxy_config: ProxyConfig | None
+ key_socket_options: _TYPE_SOCKET_OPTIONS | None
+ key__socks_options: frozenset[tuple[str, str]] | None
+ key_assert_hostname: bool | str | None
+ key_assert_fingerprint: str | None
+ key_server_hostname: str | None
+ key_blocksize: int | None
+
+
+def _default_key_normalizer(
+ key_class: type[PoolKey], request_context: dict[str, typing.Any]
+) -> PoolKey:
+ """
+ Create a pool key out of a request context dictionary.
+
+ According to RFC 3986, both the scheme and host are case-insensitive.
+ Therefore, this function normalizes both before constructing the pool
+ key for an HTTPS request. If you wish to change this behaviour, provide
+ alternate callables to ``key_fn_by_scheme``.
+
+ :param key_class:
+ The class to use when constructing the key. This should be a namedtuple
+ with the ``scheme`` and ``host`` keys at a minimum.
+ :type key_class: namedtuple
+ :param request_context:
+ A dictionary-like object that contain the context for a request.
+ :type request_context: dict
+
+ :return: A namedtuple that can be used as a connection pool key.
+ :rtype: PoolKey
+ """
+ # Since we mutate the dictionary, make a copy first
+ context = request_context.copy()
+ context["scheme"] = context["scheme"].lower()
+ context["host"] = context["host"].lower()
+
+ # These are both dictionaries and need to be transformed into frozensets
+ for key in ("headers", "_proxy_headers", "_socks_options"):
+ if key in context and context[key] is not None:
+ context[key] = frozenset(context[key].items())
+
+ # The socket_options key may be a list and needs to be transformed into a
+ # tuple.
+ socket_opts = context.get("socket_options")
+ if socket_opts is not None:
+ context["socket_options"] = tuple(socket_opts)
+
+ # Map the kwargs to the names in the namedtuple - this is necessary since
+ # namedtuples can't have fields starting with '_'.
+ for key in list(context.keys()):
+ context["key_" + key] = context.pop(key)
+
+ # Default to ``None`` for keys missing from the context
+ for field in key_class._fields:
+ if field not in context:
+ context[field] = None
+
+ # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
+ if context.get("key_blocksize") is None:
+ context["key_blocksize"] = _DEFAULT_BLOCKSIZE
+
+ return key_class(**context)
+
+
+#: A dictionary that maps a scheme to a callable that creates a pool key.
+#: This can be used to alter the way pool keys are constructed, if desired.
+#: Each PoolManager makes a copy of this dictionary so they can be configured
+#: globally here, or individually on the instance.
+key_fn_by_scheme = {
+ "http": functools.partial(_default_key_normalizer, PoolKey),
+ "https": functools.partial(_default_key_normalizer, PoolKey),
+}
+
+pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
+
+
+class PoolManager(RequestMethods):
+ """
+ Allows for arbitrary requests while transparently keeping track of
+ necessary connection pools for you.
+
+ :param num_pools:
+ Number of connection pools to cache before discarding the least
+ recently used pool.
+
+ :param headers:
+ Headers to include with all requests, unless other headers are given
+ explicitly.
+
+ :param \\**connection_pool_kw:
+ Additional parameters are used to create fresh
+ :class:`urllib3.connectionpool.ConnectionPool` instances.
+
+ Example:
+
+ .. code-block:: python
+
+ import urllib3
+
+ http = urllib3.PoolManager(num_pools=2)
+
+ resp1 = http.request("GET", "https://google.com/")
+ resp2 = http.request("GET", "https://google.com/mail")
+ resp3 = http.request("GET", "https://yahoo.com/")
+
+ print(len(http.pools))
+ # 2
+
+ """
+
+ proxy: Url | None = None
+ proxy_config: ProxyConfig | None = None
+
+ def __init__(
+ self,
+ num_pools: int = 10,
+ headers: typing.Mapping[str, str] | None = None,
+ **connection_pool_kw: typing.Any,
+ ) -> None:
+ super().__init__(headers)
+ if "retries" in connection_pool_kw:
+ retries = connection_pool_kw["retries"]
+ if not isinstance(retries, Retry):
+ # When Retry is initialized, raise_on_redirect is based
+ # on a redirect boolean value.
+ # But requests made via a pool manager always set
+ # redirect to False, and raise_on_redirect always ends
+ # up being False consequently.
+ # Here we fix the issue by setting raise_on_redirect to
+ # a value needed by the pool manager without considering
+ # the redirect boolean.
+ raise_on_redirect = retries is not False
+ retries = Retry.from_int(retries, redirect=False)
+ retries.raise_on_redirect = raise_on_redirect
+ connection_pool_kw = connection_pool_kw.copy()
+ connection_pool_kw["retries"] = retries
+ self.connection_pool_kw = connection_pool_kw
+
+ self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
+ self.pools = RecentlyUsedContainer(num_pools)
+
+ # Locally set the pool classes and keys so other PoolManagers can
+ # override them.
+ self.pool_classes_by_scheme = pool_classes_by_scheme
+ self.key_fn_by_scheme = key_fn_by_scheme.copy()
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> typing.Literal[False]:
+ self.clear()
+ # Return False to re-raise any potential exceptions
+ return False
+
+ def _new_pool(
+ self,
+ scheme: str,
+ host: str,
+ port: int,
+ request_context: dict[str, typing.Any] | None = None,
+ ) -> HTTPConnectionPool:
+ """
+ Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
+ any additional pool keyword arguments.
+
+ If ``request_context`` is provided, it is provided as keyword arguments
+ to the pool class used. This method is used to actually create the
+ connection pools handed out by :meth:`connection_from_url` and
+ companion methods. It is intended to be overridden for customization.
+ """
+ pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
+ if request_context is None:
+ request_context = self.connection_pool_kw.copy()
+
+ # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
+ # set to 'None' in the request_context.
+ if request_context.get("blocksize") is None:
+ request_context["blocksize"] = _DEFAULT_BLOCKSIZE
+
+ # Although the context has everything necessary to create the pool,
+ # this function has historically only used the scheme, host, and port
+ # in the positional args. When an API change is acceptable these can
+ # be removed.
+ for key in ("scheme", "host", "port"):
+ request_context.pop(key, None)
+
+ if scheme == "http":
+ for kw in SSL_KEYWORDS:
+ request_context.pop(kw, None)
+
+ return pool_cls(host, port, **request_context)
+
+ def clear(self) -> None:
+ """
+ Empty our store of pools and direct them all to close.
+
+ This will not affect in-flight connections, but they will not be
+ re-used after completion.
+ """
+ self.pools.clear()
+
+ def connection_from_host(
+ self,
+ host: str | None,
+ port: int | None = None,
+ scheme: str | None = "http",
+ pool_kwargs: dict[str, typing.Any] | None = None,
+ ) -> HTTPConnectionPool:
+ """
+ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
+
+ If ``port`` isn't given, it will be derived from the ``scheme`` using
+ ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
+ provided, it is merged with the instance's ``connection_pool_kw``
+ variable and used to create the new connection pool, if one is
+ needed.
+ """
+
+ if not host:
+ raise LocationValueError("No host specified.")
+
+ request_context = self._merge_pool_kwargs(pool_kwargs)
+ request_context["scheme"] = scheme or "http"
+ if not port:
+ port = port_by_scheme.get(request_context["scheme"].lower(), 80)
+ request_context["port"] = port
+ request_context["host"] = host
+
+ return self.connection_from_context(request_context)
+
+ def connection_from_context(
+ self, request_context: dict[str, typing.Any]
+ ) -> HTTPConnectionPool:
+ """
+ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
+
+ ``request_context`` must at least contain the ``scheme`` key and its
+ value must be a key in ``key_fn_by_scheme`` instance variable.
+ """
+ if "strict" in request_context:
+ warnings.warn(
+ "The 'strict' parameter is no longer needed on Python 3+. "
+ "This will raise an error in urllib3 v2.1.0.",
+ DeprecationWarning,
+ )
+ request_context.pop("strict")
+
+ scheme = request_context["scheme"].lower()
+ pool_key_constructor = self.key_fn_by_scheme.get(scheme)
+ if not pool_key_constructor:
+ raise URLSchemeUnknown(scheme)
+ pool_key = pool_key_constructor(request_context)
+
+ return self.connection_from_pool_key(pool_key, request_context=request_context)
+
+ def connection_from_pool_key(
+ self, pool_key: PoolKey, request_context: dict[str, typing.Any]
+ ) -> HTTPConnectionPool:
+ """
+ Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
+
+ ``pool_key`` should be a namedtuple that only contains immutable
+ objects. At a minimum it must have the ``scheme``, ``host``, and
+ ``port`` fields.
+ """
+ with self.pools.lock:
+ # If the scheme, host, or port doesn't match existing open
+ # connections, open a new ConnectionPool.
+ pool = self.pools.get(pool_key)
+ if pool:
+ return pool
+
+ # Make a fresh ConnectionPool of the desired type
+ scheme = request_context["scheme"]
+ host = request_context["host"]
+ port = request_context["port"]
+ pool = self._new_pool(scheme, host, port, request_context=request_context)
+ self.pools[pool_key] = pool
+
+ return pool
+
+ def connection_from_url(
+ self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
+ ) -> HTTPConnectionPool:
+ """
+ Similar to :func:`urllib3.connectionpool.connection_from_url`.
+
+ If ``pool_kwargs`` is not provided and a new pool needs to be
+ constructed, ``self.connection_pool_kw`` is used to initialize
+ the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
+ is provided, it is used instead. Note that if a new pool does not
+ need to be created for the request, the provided ``pool_kwargs`` are
+ not used.
+ """
+ u = parse_url(url)
+ return self.connection_from_host(
+ u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
+ )
+
+ def _merge_pool_kwargs(
+ self, override: dict[str, typing.Any] | None
+ ) -> dict[str, typing.Any]:
+ """
+ Merge a dictionary of override values for self.connection_pool_kw.
+
+ This does not modify self.connection_pool_kw and returns a new dict.
+ Any keys in the override dictionary with a value of ``None`` are
+ removed from the merged dictionary.
+ """
+ base_pool_kwargs = self.connection_pool_kw.copy()
+ if override:
+ for key, value in override.items():
+ if value is None:
+ try:
+ del base_pool_kwargs[key]
+ except KeyError:
+ pass
+ else:
+ base_pool_kwargs[key] = value
+ return base_pool_kwargs
+
+ def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
+ """
+ Indicates if the proxy requires the complete destination URL in the
+ request. Normally this is only needed when not using an HTTP CONNECT
+ tunnel.
+ """
+ if self.proxy is None:
+ return False
+
+ return not connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, parsed_url.scheme
+ )
+
+ def urlopen( # type: ignore[override]
+ self, method: str, url: str, redirect: bool = True, **kw: typing.Any
+ ) -> BaseHTTPResponse:
+ """
+ Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
+ with custom cross-host redirect logic and only sends the request-uri
+ portion of the ``url``.
+
+ The given ``url`` parameter must be absolute, such that an appropriate
+ :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
+ """
+ u = parse_url(url)
+
+ if u.scheme is None:
+ warnings.warn(
+ "URLs without a scheme (ie 'https://') are deprecated and will raise an error "
+ "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs "
+ "start with 'https://' or 'http://'. Read more in this issue: "
+ "https://github.com/urllib3/urllib3/issues/2920",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+
+ conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
+
+ kw["assert_same_host"] = False
+ kw["redirect"] = False
+
+ if "headers" not in kw:
+ kw["headers"] = self.headers
+
+ if self._proxy_requires_url_absolute_form(u):
+ response = conn.urlopen(method, url, **kw)
+ else:
+ response = conn.urlopen(method, u.request_uri, **kw)
+
+ redirect_location = redirect and response.get_redirect_location()
+ if not redirect_location:
+ return response
+
+ # Support relative URLs for redirecting.
+ redirect_location = urljoin(url, redirect_location)
+
+ if response.status == 303:
+ # Change the method according to RFC 9110, Section 15.4.4.
+ method = "GET"
+ # And lose the body not to transfer anything sensitive.
+ kw["body"] = None
+ kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
+
+ retries = kw.get("retries", response.retries)
+ if not isinstance(retries, Retry):
+ retries = Retry.from_int(retries, redirect=redirect)
+
+ # Strip headers marked as unsafe to forward to the redirected location.
+ # Check remove_headers_on_redirect to avoid a potential network call within
+ # conn.is_same_host() which may use socket.gethostbyname() in the future.
+ if retries.remove_headers_on_redirect and not conn.is_same_host(
+ redirect_location
+ ):
+ new_headers = kw["headers"].copy()
+ for header in kw["headers"]:
+ if header.lower() in retries.remove_headers_on_redirect:
+ new_headers.pop(header, None)
+ kw["headers"] = new_headers
+
+ try:
+ retries = retries.increment(method, url, response=response, _pool=conn)
+ except MaxRetryError:
+ if retries.raise_on_redirect:
+ response.drain_conn()
+ raise
+ return response
+
+ kw["retries"] = retries
+ kw["redirect"] = redirect
+
+ log.info("Redirecting %s -> %s", url, redirect_location)
+
+ response.drain_conn()
+ return self.urlopen(method, redirect_location, **kw)
+
+
+class ProxyManager(PoolManager):
+ """
+ Behaves just like :class:`PoolManager`, but sends all requests through
+ the defined proxy, using the CONNECT method for HTTPS URLs.
+
+ :param proxy_url:
+ The URL of the proxy to be used.
+
+ :param proxy_headers:
+ A dictionary containing headers that will be sent to the proxy. In case
+ of HTTP they are being sent with each request, while in the
+ HTTPS/CONNECT case they are sent only once. Could be used for proxy
+ authentication.
+
+ :param proxy_ssl_context:
+ The proxy SSL context is used to establish the TLS connection to the
+ proxy when using HTTPS proxies.
+
+ :param use_forwarding_for_https:
+ (Defaults to False) If set to True will forward requests to the HTTPS
+ proxy to be made on behalf of the client instead of creating a TLS
+ tunnel via the CONNECT method. **Enabling this flag means that request
+ and response headers and content will be visible from the HTTPS proxy**
+ whereas tunneling keeps request and response headers and content
+ private. IP address, target hostname, SNI, and port are always visible
+ to an HTTPS proxy even when this flag is disabled.
+
+ :param proxy_assert_hostname:
+ The hostname of the certificate to verify against.
+
+ :param proxy_assert_fingerprint:
+ The fingerprint of the certificate to verify against.
+
+ Example:
+
+ .. code-block:: python
+
+ import urllib3
+
+ proxy = urllib3.ProxyManager("https://localhost:3128/")
+
+ resp1 = proxy.request("GET", "https://google.com/")
+ resp2 = proxy.request("GET", "https://httpbin.org/")
+
+ print(len(proxy.pools))
+ # 1
+
+ resp3 = proxy.request("GET", "https://httpbin.org/")
+ resp4 = proxy.request("GET", "https://twitter.com/")
+
+ print(len(proxy.pools))
+ # 3
+
+ """
+
+ def __init__(
+ self,
+ proxy_url: str,
+ num_pools: int = 10,
+ headers: typing.Mapping[str, str] | None = None,
+ proxy_headers: typing.Mapping[str, str] | None = None,
+ proxy_ssl_context: ssl.SSLContext | None = None,
+ use_forwarding_for_https: bool = False,
+ proxy_assert_hostname: None | str | typing.Literal[False] = None,
+ proxy_assert_fingerprint: str | None = None,
+ **connection_pool_kw: typing.Any,
+ ) -> None:
+ if isinstance(proxy_url, HTTPConnectionPool):
+ str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
+ else:
+ str_proxy_url = proxy_url
+ proxy = parse_url(str_proxy_url)
+
+ if proxy.scheme not in ("http", "https"):
+ raise ProxySchemeUnknown(proxy.scheme)
+
+ if not proxy.port:
+ port = port_by_scheme.get(proxy.scheme, 80)
+ proxy = proxy._replace(port=port)
+
+ self.proxy = proxy
+ self.proxy_headers = proxy_headers or {}
+ self.proxy_ssl_context = proxy_ssl_context
+ self.proxy_config = ProxyConfig(
+ proxy_ssl_context,
+ use_forwarding_for_https,
+ proxy_assert_hostname,
+ proxy_assert_fingerprint,
+ )
+
+ connection_pool_kw["_proxy"] = self.proxy
+ connection_pool_kw["_proxy_headers"] = self.proxy_headers
+ connection_pool_kw["_proxy_config"] = self.proxy_config
+
+ super().__init__(num_pools, headers, **connection_pool_kw)
+
+ def connection_from_host(
+ self,
+ host: str | None,
+ port: int | None = None,
+ scheme: str | None = "http",
+ pool_kwargs: dict[str, typing.Any] | None = None,
+ ) -> HTTPConnectionPool:
+ if scheme == "https":
+ return super().connection_from_host(
+ host, port, scheme, pool_kwargs=pool_kwargs
+ )
+
+ return super().connection_from_host(
+ self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
+ )
+
+ def _set_proxy_headers(
+ self, url: str, headers: typing.Mapping[str, str] | None = None
+ ) -> typing.Mapping[str, str]:
+ """
+ Sets headers needed by proxies: specifically, the Accept and Host
+ headers. Only sets headers not provided by the user.
+ """
+ headers_ = {"Accept": "*/*"}
+
+ netloc = parse_url(url).netloc
+ if netloc:
+ headers_["Host"] = netloc
+
+ if headers:
+ headers_.update(headers)
+ return headers_
+
+ def urlopen( # type: ignore[override]
+ self, method: str, url: str, redirect: bool = True, **kw: typing.Any
+ ) -> BaseHTTPResponse:
+ "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
+ u = parse_url(url)
+ if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
+ # For connections using HTTP CONNECT, httplib sets the necessary
+ # headers on the CONNECT to the proxy. If we're not using CONNECT,
+ # we'll definitely need to set 'Host' at the very least.
+ headers = kw.get("headers", self.headers)
+ kw["headers"] = self._set_proxy_headers(url, headers)
+
+ return super().urlopen(method, url, redirect=redirect, **kw)
+
+
+def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
+ return ProxyManager(proxy_url=url, **kw)
diff --git a/phivenv/Lib/site-packages/urllib3/py.typed b/phivenv/Lib/site-packages/urllib3/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..5f3ea3d919363f08ab03edbc85b6099bc4df5647
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/py.typed
@@ -0,0 +1,2 @@
+# Instruct type checkers to look for inline type annotations in this package.
+# See PEP 561.
diff --git a/phivenv/Lib/site-packages/urllib3/response.py b/phivenv/Lib/site-packages/urllib3/response.py
new file mode 100644
index 0000000000000000000000000000000000000000..5632dab3b2f93df91bf132384c3820a6b5518f43
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/response.py
@@ -0,0 +1,1307 @@
+from __future__ import annotations
+
+import collections
+import io
+import json as _json
+import logging
+import re
+import socket
+import sys
+import typing
+import warnings
+import zlib
+from contextlib import contextmanager
+from http.client import HTTPMessage as _HttplibHTTPMessage
+from http.client import HTTPResponse as _HttplibHTTPResponse
+from socket import timeout as SocketTimeout
+
+if typing.TYPE_CHECKING:
+ from ._base_connection import BaseHTTPConnection
+
+try:
+ try:
+ import brotlicffi as brotli # type: ignore[import-not-found]
+ except ImportError:
+ import brotli # type: ignore[import-not-found]
+except ImportError:
+ brotli = None
+
+from . import util
+from ._base_connection import _TYPE_BODY
+from ._collections import HTTPHeaderDict
+from .connection import BaseSSLError, HTTPConnection, HTTPException
+from .exceptions import (
+ BodyNotHttplibCompatible,
+ DecodeError,
+ HTTPError,
+ IncompleteRead,
+ InvalidChunkLength,
+ InvalidHeader,
+ ProtocolError,
+ ReadTimeoutError,
+ ResponseNotChunked,
+ SSLError,
+)
+from .util.response import is_fp_closed, is_response_to_head
+from .util.retry import Retry
+
+if typing.TYPE_CHECKING:
+ from .connectionpool import HTTPConnectionPool
+
+log = logging.getLogger(__name__)
+
+
+class ContentDecoder:
+ def decompress(self, data: bytes) -> bytes:
+ raise NotImplementedError()
+
+ def flush(self) -> bytes:
+ raise NotImplementedError()
+
+
+class DeflateDecoder(ContentDecoder):
+ def __init__(self) -> None:
+ self._first_try = True
+ self._data = b""
+ self._obj = zlib.decompressobj()
+
+ def decompress(self, data: bytes) -> bytes:
+ if not data:
+ return data
+
+ if not self._first_try:
+ return self._obj.decompress(data)
+
+ self._data += data
+ try:
+ decompressed = self._obj.decompress(data)
+ if decompressed:
+ self._first_try = False
+ self._data = None # type: ignore[assignment]
+ return decompressed
+ except zlib.error:
+ self._first_try = False
+ self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
+ try:
+ return self.decompress(self._data)
+ finally:
+ self._data = None # type: ignore[assignment]
+
+ def flush(self) -> bytes:
+ return self._obj.flush()
+
+
+class GzipDecoderState:
+ FIRST_MEMBER = 0
+ OTHER_MEMBERS = 1
+ SWALLOW_DATA = 2
+
+
+class GzipDecoder(ContentDecoder):
+ def __init__(self) -> None:
+ self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+ self._state = GzipDecoderState.FIRST_MEMBER
+
+ def decompress(self, data: bytes) -> bytes:
+ ret = bytearray()
+ if self._state == GzipDecoderState.SWALLOW_DATA or not data:
+ return bytes(ret)
+ while True:
+ try:
+ ret += self._obj.decompress(data)
+ except zlib.error:
+ previous_state = self._state
+ # Ignore data after the first error
+ self._state = GzipDecoderState.SWALLOW_DATA
+ if previous_state == GzipDecoderState.OTHER_MEMBERS:
+ # Allow trailing garbage acceptable in other gzip clients
+ return bytes(ret)
+ raise
+ data = self._obj.unused_data
+ if not data:
+ return bytes(ret)
+ self._state = GzipDecoderState.OTHER_MEMBERS
+ self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+ def flush(self) -> bytes:
+ return self._obj.flush()
+
+
+if brotli is not None:
+
+ class BrotliDecoder(ContentDecoder):
+ # Supports both 'brotlipy' and 'Brotli' packages
+ # since they share an import name. The top branches
+ # are for 'brotlipy' and bottom branches for 'Brotli'
+ def __init__(self) -> None:
+ self._obj = brotli.Decompressor()
+ if hasattr(self._obj, "decompress"):
+ setattr(self, "decompress", self._obj.decompress)
+ else:
+ setattr(self, "decompress", self._obj.process)
+
+ def flush(self) -> bytes:
+ if hasattr(self._obj, "flush"):
+ return self._obj.flush() # type: ignore[no-any-return]
+ return b""
+
+
+try:
+ # Python 3.14+
+ from compression import zstd # type: ignore[import-not-found] # noqa: F401
+
+ HAS_ZSTD = True
+
+ class ZstdDecoder(ContentDecoder):
+ def __init__(self) -> None:
+ self._obj = zstd.ZstdDecompressor()
+
+ def decompress(self, data: bytes) -> bytes:
+ if not data:
+ return b""
+ data_parts = [self._obj.decompress(data)]
+ while self._obj.eof and self._obj.unused_data:
+ unused_data = self._obj.unused_data
+ self._obj = zstd.ZstdDecompressor()
+ data_parts.append(self._obj.decompress(unused_data))
+ return b"".join(data_parts)
+
+ def flush(self) -> bytes:
+ if not self._obj.eof:
+ raise DecodeError("Zstandard data is incomplete")
+ return b""
+
+except ImportError:
+ try:
+ # Python 3.13 and earlier require the 'zstandard' module.
+ import zstandard as zstd
+
+ # The package 'zstandard' added the 'eof' property starting
+ # in v0.18.0 which we require to ensure a complete and
+ # valid zstd stream was fed into the ZstdDecoder.
+ # See: https://github.com/urllib3/urllib3/pull/2624
+ _zstd_version = tuple(
+ map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr]
+ )
+ if _zstd_version < (0, 18): # Defensive:
+ raise ImportError("zstandard module doesn't have eof")
+ except (AttributeError, ImportError, ValueError): # Defensive:
+ HAS_ZSTD = False
+ else:
+ HAS_ZSTD = True
+
+ class ZstdDecoder(ContentDecoder): # type: ignore[no-redef]
+ def __init__(self) -> None:
+ self._obj = zstd.ZstdDecompressor().decompressobj()
+
+ def decompress(self, data: bytes) -> bytes:
+ if not data:
+ return b""
+ data_parts = [self._obj.decompress(data)]
+ while self._obj.eof and self._obj.unused_data:
+ unused_data = self._obj.unused_data
+ self._obj = zstd.ZstdDecompressor().decompressobj()
+ data_parts.append(self._obj.decompress(unused_data))
+ return b"".join(data_parts)
+
+ def flush(self) -> bytes:
+ ret = self._obj.flush() # note: this is a no-op
+ if not self._obj.eof:
+ raise DecodeError("Zstandard data is incomplete")
+ return ret # type: ignore[no-any-return]
+
+
+class MultiDecoder(ContentDecoder):
+ """
+ From RFC7231:
+ If one or more encodings have been applied to a representation, the
+ sender that applied the encodings MUST generate a Content-Encoding
+ header field that lists the content codings in the order in which
+ they were applied.
+ """
+
+ def __init__(self, modes: str) -> None:
+ self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
+
+ def flush(self) -> bytes:
+ return self._decoders[0].flush()
+
+ def decompress(self, data: bytes) -> bytes:
+ for d in reversed(self._decoders):
+ data = d.decompress(data)
+ return data
+
+
+def _get_decoder(mode: str) -> ContentDecoder:
+ if "," in mode:
+ return MultiDecoder(mode)
+
+ # According to RFC 9110 section 8.4.1.3, recipients should
+ # consider x-gzip equivalent to gzip
+ if mode in ("gzip", "x-gzip"):
+ return GzipDecoder()
+
+ if brotli is not None and mode == "br":
+ return BrotliDecoder()
+
+ if HAS_ZSTD and mode == "zstd":
+ return ZstdDecoder()
+
+ return DeflateDecoder()
+
+
+class BytesQueueBuffer:
+ """Memory-efficient bytes buffer
+
+ To return decoded data in read() and still follow the BufferedIOBase API, we need a
+ buffer to always return the correct amount of bytes.
+
+ This buffer should be filled using calls to put()
+
+ Our maximum memory usage is determined by the sum of the size of:
+
+ * self.buffer, which contains the full data
+ * the largest chunk that we will copy in get()
+
+ The worst case scenario is a single chunk, in which case we'll make a full copy of
+ the data inside get().
+ """
+
+ def __init__(self) -> None:
+ self.buffer: typing.Deque[bytes] = collections.deque()
+ self._size: int = 0
+
+ def __len__(self) -> int:
+ return self._size
+
+ def put(self, data: bytes) -> None:
+ self.buffer.append(data)
+ self._size += len(data)
+
+ def get(self, n: int) -> bytes:
+ if n == 0:
+ return b""
+ elif not self.buffer:
+ raise RuntimeError("buffer is empty")
+ elif n < 0:
+ raise ValueError("n should be > 0")
+
+ fetched = 0
+ ret = io.BytesIO()
+ while fetched < n:
+ remaining = n - fetched
+ chunk = self.buffer.popleft()
+ chunk_length = len(chunk)
+ if remaining < chunk_length:
+ left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
+ ret.write(left_chunk)
+ self.buffer.appendleft(right_chunk)
+ self._size -= remaining
+ break
+ else:
+ ret.write(chunk)
+ self._size -= chunk_length
+ fetched += chunk_length
+
+ if not self.buffer:
+ break
+
+ return ret.getvalue()
+
+ def get_all(self) -> bytes:
+ buffer = self.buffer
+ if not buffer:
+ assert self._size == 0
+ return b""
+ if len(buffer) == 1:
+ result = buffer.pop()
+ else:
+ ret = io.BytesIO()
+ ret.writelines(buffer.popleft() for _ in range(len(buffer)))
+ result = ret.getvalue()
+ self._size = 0
+ return result
+
+
+class BaseHTTPResponse(io.IOBase):
+ CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
+ if brotli is not None:
+ CONTENT_DECODERS += ["br"]
+ if HAS_ZSTD:
+ CONTENT_DECODERS += ["zstd"]
+ REDIRECT_STATUSES = [301, 302, 303, 307, 308]
+
+ DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
+ if brotli is not None:
+ DECODER_ERROR_CLASSES += (brotli.error,)
+
+ if HAS_ZSTD:
+ DECODER_ERROR_CLASSES += (zstd.ZstdError,)
+
+ def __init__(
+ self,
+ *,
+ headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
+ status: int,
+ version: int,
+ version_string: str,
+ reason: str | None,
+ decode_content: bool,
+ request_url: str | None,
+ retries: Retry | None = None,
+ ) -> None:
+ if isinstance(headers, HTTPHeaderDict):
+ self.headers = headers
+ else:
+ self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type]
+ self.status = status
+ self.version = version
+ self.version_string = version_string
+ self.reason = reason
+ self.decode_content = decode_content
+ self._has_decoded_content = False
+ self._request_url: str | None = request_url
+ self.retries = retries
+
+ self.chunked = False
+ tr_enc = self.headers.get("transfer-encoding", "").lower()
+ # Don't incur the penalty of creating a list and then discarding it
+ encodings = (enc.strip() for enc in tr_enc.split(","))
+ if "chunked" in encodings:
+ self.chunked = True
+
+ self._decoder: ContentDecoder | None = None
+ self.length_remaining: int | None
+
+ def get_redirect_location(self) -> str | None | typing.Literal[False]:
+ """
+ Should we redirect and where to?
+
+ :returns: Truthy redirect location string if we got a redirect status
+ code and valid location. ``None`` if redirect status and no
+ location. ``False`` if not a redirect status code.
+ """
+ if self.status in self.REDIRECT_STATUSES:
+ return self.headers.get("location")
+ return False
+
+ @property
+ def data(self) -> bytes:
+ raise NotImplementedError()
+
+ def json(self) -> typing.Any:
+ """
+ Deserializes the body of the HTTP response as a Python object.
+
+ The body of the HTTP response must be encoded using UTF-8, as per
+ `RFC 8529 Section 8.1 `_.
+
+ To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
+ your custom decoder instead.
+
+ If the body of the HTTP response is not decodable to UTF-8, a
+ `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
+ valid JSON document, a `json.JSONDecodeError` will be raised.
+
+ Read more :ref:`here `.
+
+ :returns: The body of the HTTP response as a Python object.
+ """
+ data = self.data.decode("utf-8")
+ return _json.loads(data)
+
+ @property
+ def url(self) -> str | None:
+ raise NotImplementedError()
+
+ @url.setter
+ def url(self, url: str | None) -> None:
+ raise NotImplementedError()
+
+ @property
+ def connection(self) -> BaseHTTPConnection | None:
+ raise NotImplementedError()
+
+ @property
+ def retries(self) -> Retry | None:
+ return self._retries
+
+ @retries.setter
+ def retries(self, retries: Retry | None) -> None:
+ # Override the request_url if retries has a redirect location.
+ if retries is not None and retries.history:
+ self.url = retries.history[-1].redirect_location
+ self._retries = retries
+
+ def stream(
+ self, amt: int | None = 2**16, decode_content: bool | None = None
+ ) -> typing.Iterator[bytes]:
+ raise NotImplementedError()
+
+ def read(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ cache_content: bool = False,
+ ) -> bytes:
+ raise NotImplementedError()
+
+ def read1(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ ) -> bytes:
+ raise NotImplementedError()
+
+ def read_chunked(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ ) -> typing.Iterator[bytes]:
+ raise NotImplementedError()
+
+ def release_conn(self) -> None:
+ raise NotImplementedError()
+
+ def drain_conn(self) -> None:
+ raise NotImplementedError()
+
+ def shutdown(self) -> None:
+ raise NotImplementedError()
+
+ def close(self) -> None:
+ raise NotImplementedError()
+
+ def _init_decoder(self) -> None:
+ """
+ Set-up the _decoder attribute if necessary.
+ """
+ # Note: content-encoding value should be case-insensitive, per RFC 7230
+ # Section 3.2
+ content_encoding = self.headers.get("content-encoding", "").lower()
+ if self._decoder is None:
+ if content_encoding in self.CONTENT_DECODERS:
+ self._decoder = _get_decoder(content_encoding)
+ elif "," in content_encoding:
+ encodings = [
+ e.strip()
+ for e in content_encoding.split(",")
+ if e.strip() in self.CONTENT_DECODERS
+ ]
+ if encodings:
+ self._decoder = _get_decoder(content_encoding)
+
+ def _decode(
+ self, data: bytes, decode_content: bool | None, flush_decoder: bool
+ ) -> bytes:
+ """
+ Decode the data passed in and potentially flush the decoder.
+ """
+ if not decode_content:
+ if self._has_decoded_content:
+ raise RuntimeError(
+ "Calling read(decode_content=False) is not supported after "
+ "read(decode_content=True) was called."
+ )
+ return data
+
+ try:
+ if self._decoder:
+ data = self._decoder.decompress(data)
+ self._has_decoded_content = True
+ except self.DECODER_ERROR_CLASSES as e:
+ content_encoding = self.headers.get("content-encoding", "").lower()
+ raise DecodeError(
+ "Received response with content-encoding: %s, but "
+ "failed to decode it." % content_encoding,
+ e,
+ ) from e
+ if flush_decoder:
+ data += self._flush_decoder()
+
+ return data
+
+ def _flush_decoder(self) -> bytes:
+ """
+ Flushes the decoder. Should only be called if the decoder is actually
+ being used.
+ """
+ if self._decoder:
+ return self._decoder.decompress(b"") + self._decoder.flush()
+ return b""
+
+ # Compatibility methods for `io` module
+ def readinto(self, b: bytearray) -> int:
+ temp = self.read(len(b))
+ if len(temp) == 0:
+ return 0
+ else:
+ b[: len(temp)] = temp
+ return len(temp)
+
+ # Compatibility methods for http.client.HTTPResponse
+ def getheaders(self) -> HTTPHeaderDict:
+ warnings.warn(
+ "HTTPResponse.getheaders() is deprecated and will be removed "
+ "in urllib3 v2.6.0. Instead access HTTPResponse.headers directly.",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+ return self.headers
+
+ def getheader(self, name: str, default: str | None = None) -> str | None:
+ warnings.warn(
+ "HTTPResponse.getheader() is deprecated and will be removed "
+ "in urllib3 v2.6.0. Instead use HTTPResponse.headers.get(name, default).",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+ return self.headers.get(name, default)
+
+ # Compatibility method for http.cookiejar
+ def info(self) -> HTTPHeaderDict:
+ return self.headers
+
+ def geturl(self) -> str | None:
+ return self.url
+
+
+class HTTPResponse(BaseHTTPResponse):
+ """
+ HTTP Response container.
+
+ Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
+ loaded and decoded on-demand when the ``data`` property is accessed. This
+ class is also compatible with the Python standard library's :mod:`io`
+ module, and can hence be treated as a readable object in the context of that
+ framework.
+
+ Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
+
+ :param preload_content:
+ If True, the response's body will be preloaded during construction.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param original_response:
+ When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
+ object, it's convenient to include the original for debug purposes. It's
+ otherwise unused.
+
+ :param retries:
+ The retries contains the last :class:`~urllib3.util.retry.Retry` that
+ was used during the request.
+
+ :param enforce_content_length:
+ Enforce content length checking. Body returned by server must match
+ value of Content-Length header, if present. Otherwise, raise error.
+ """
+
+ def __init__(
+ self,
+ body: _TYPE_BODY = "",
+ headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
+ status: int = 0,
+ version: int = 0,
+ version_string: str = "HTTP/?",
+ reason: str | None = None,
+ preload_content: bool = True,
+ decode_content: bool = True,
+ original_response: _HttplibHTTPResponse | None = None,
+ pool: HTTPConnectionPool | None = None,
+ connection: HTTPConnection | None = None,
+ msg: _HttplibHTTPMessage | None = None,
+ retries: Retry | None = None,
+ enforce_content_length: bool = True,
+ request_method: str | None = None,
+ request_url: str | None = None,
+ auto_close: bool = True,
+ sock_shutdown: typing.Callable[[int], None] | None = None,
+ ) -> None:
+ super().__init__(
+ headers=headers,
+ status=status,
+ version=version,
+ version_string=version_string,
+ reason=reason,
+ decode_content=decode_content,
+ request_url=request_url,
+ retries=retries,
+ )
+
+ self.enforce_content_length = enforce_content_length
+ self.auto_close = auto_close
+
+ self._body = None
+ self._fp: _HttplibHTTPResponse | None = None
+ self._original_response = original_response
+ self._fp_bytes_read = 0
+ self.msg = msg
+
+ if body and isinstance(body, (str, bytes)):
+ self._body = body
+
+ self._pool = pool
+ self._connection = connection
+
+ if hasattr(body, "read"):
+ self._fp = body # type: ignore[assignment]
+ self._sock_shutdown = sock_shutdown
+
+ # Are we using the chunked-style of transfer encoding?
+ self.chunk_left: int | None = None
+
+ # Determine length of response
+ self.length_remaining = self._init_length(request_method)
+
+ # Used to return the correct amount of bytes for partial read()s
+ self._decoded_buffer = BytesQueueBuffer()
+
+ # If requested, preload the body.
+ if preload_content and not self._body:
+ self._body = self.read(decode_content=decode_content)
+
+ def release_conn(self) -> None:
+ if not self._pool or not self._connection:
+ return None
+
+ self._pool._put_conn(self._connection)
+ self._connection = None
+
+ def drain_conn(self) -> None:
+ """
+ Read and discard any remaining HTTP response data in the response connection.
+
+ Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
+ """
+ try:
+ self.read()
+ except (HTTPError, OSError, BaseSSLError, HTTPException):
+ pass
+
+ @property
+ def data(self) -> bytes:
+ # For backwards-compat with earlier urllib3 0.4 and earlier.
+ if self._body:
+ return self._body # type: ignore[return-value]
+
+ if self._fp:
+ return self.read(cache_content=True)
+
+ return None # type: ignore[return-value]
+
+ @property
+ def connection(self) -> HTTPConnection | None:
+ return self._connection
+
+ def isclosed(self) -> bool:
+ return is_fp_closed(self._fp)
+
+ def tell(self) -> int:
+ """
+ Obtain the number of bytes pulled over the wire so far. May differ from
+ the amount of content returned by :meth:``urllib3.response.HTTPResponse.read``
+ if bytes are encoded on the wire (e.g, compressed).
+ """
+ return self._fp_bytes_read
+
+ def _init_length(self, request_method: str | None) -> int | None:
+ """
+ Set initial length value for Response content if available.
+ """
+ length: int | None
+ content_length: str | None = self.headers.get("content-length")
+
+ if content_length is not None:
+ if self.chunked:
+ # This Response will fail with an IncompleteRead if it can't be
+ # received as chunked. This method falls back to attempt reading
+ # the response before raising an exception.
+ log.warning(
+ "Received response with both Content-Length and "
+ "Transfer-Encoding set. This is expressly forbidden "
+ "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
+ "attempting to process response as Transfer-Encoding: "
+ "chunked."
+ )
+ return None
+
+ try:
+ # RFC 7230 section 3.3.2 specifies multiple content lengths can
+ # be sent in a single Content-Length header
+ # (e.g. Content-Length: 42, 42). This line ensures the values
+ # are all valid ints and that as long as the `set` length is 1,
+ # all values are the same. Otherwise, the header is invalid.
+ lengths = {int(val) for val in content_length.split(",")}
+ if len(lengths) > 1:
+ raise InvalidHeader(
+ "Content-Length contained multiple "
+ "unmatching values (%s)" % content_length
+ )
+ length = lengths.pop()
+ except ValueError:
+ length = None
+ else:
+ if length < 0:
+ length = None
+
+ else: # if content_length is None
+ length = None
+
+ # Convert status to int for comparison
+ # In some cases, httplib returns a status of "_UNKNOWN"
+ try:
+ status = int(self.status)
+ except ValueError:
+ status = 0
+
+ # Check for responses that shouldn't include a body
+ if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
+ length = 0
+
+ return length
+
+ @contextmanager
+ def _error_catcher(self) -> typing.Generator[None]:
+ """
+ Catch low-level python exceptions, instead re-raising urllib3
+ variants, so that low-level exceptions are not leaked in the
+ high-level api.
+
+ On exit, release the connection back to the pool.
+ """
+ clean_exit = False
+
+ try:
+ try:
+ yield
+
+ except SocketTimeout as e:
+ # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
+ # there is yet no clean way to get at it from this context.
+ raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
+
+ except BaseSSLError as e:
+ # FIXME: Is there a better way to differentiate between SSLErrors?
+ if "read operation timed out" not in str(e):
+ # SSL errors related to framing/MAC get wrapped and reraised here
+ raise SSLError(e) from e
+
+ raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
+
+ except IncompleteRead as e:
+ if (
+ e.expected is not None
+ and e.partial is not None
+ and e.expected == -e.partial
+ ):
+ arg = "Response may not contain content."
+ else:
+ arg = f"Connection broken: {e!r}"
+ raise ProtocolError(arg, e) from e
+
+ except (HTTPException, OSError) as e:
+ raise ProtocolError(f"Connection broken: {e!r}", e) from e
+
+ # If no exception is thrown, we should avoid cleaning up
+ # unnecessarily.
+ clean_exit = True
+ finally:
+ # If we didn't terminate cleanly, we need to throw away our
+ # connection.
+ if not clean_exit:
+ # The response may not be closed but we're not going to use it
+ # anymore so close it now to ensure that the connection is
+ # released back to the pool.
+ if self._original_response:
+ self._original_response.close()
+
+ # Closing the response may not actually be sufficient to close
+ # everything, so if we have a hold of the connection close that
+ # too.
+ if self._connection:
+ self._connection.close()
+
+ # If we hold the original response but it's closed now, we should
+ # return the connection back to the pool.
+ if self._original_response and self._original_response.isclosed():
+ self.release_conn()
+
+ def _fp_read(
+ self,
+ amt: int | None = None,
+ *,
+ read1: bool = False,
+ ) -> bytes:
+ """
+ Read a response with the thought that reading the number of bytes
+ larger than can fit in a 32-bit int at a time via SSL in some
+ known cases leads to an overflow error that has to be prevented
+ if `amt` or `self.length_remaining` indicate that a problem may
+ happen.
+
+ The known cases:
+ * CPython < 3.9.7 because of a bug
+ https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900.
+ * urllib3 injected with pyOpenSSL-backed SSL-support.
+ * CPython < 3.10 only when `amt` does not fit 32-bit int.
+ """
+ assert self._fp
+ c_int_max = 2**31 - 1
+ if (
+ (amt and amt > c_int_max)
+ or (
+ amt is None
+ and self.length_remaining
+ and self.length_remaining > c_int_max
+ )
+ ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)):
+ if read1:
+ return self._fp.read1(c_int_max)
+ buffer = io.BytesIO()
+ # Besides `max_chunk_amt` being a maximum chunk size, it
+ # affects memory overhead of reading a response by this
+ # method in CPython.
+ # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
+ # chunk size that does not lead to an overflow error, but
+ # 256 MiB is a compromise.
+ max_chunk_amt = 2**28
+ while amt is None or amt != 0:
+ if amt is not None:
+ chunk_amt = min(amt, max_chunk_amt)
+ amt -= chunk_amt
+ else:
+ chunk_amt = max_chunk_amt
+ data = self._fp.read(chunk_amt)
+ if not data:
+ break
+ buffer.write(data)
+ del data # to reduce peak memory usage by `max_chunk_amt`.
+ return buffer.getvalue()
+ elif read1:
+ return self._fp.read1(amt) if amt is not None else self._fp.read1()
+ else:
+ # StringIO doesn't like amt=None
+ return self._fp.read(amt) if amt is not None else self._fp.read()
+
+ def _raw_read(
+ self,
+ amt: int | None = None,
+ *,
+ read1: bool = False,
+ ) -> bytes:
+ """
+ Reads `amt` of bytes from the socket.
+ """
+ if self._fp is None:
+ return None # type: ignore[return-value]
+
+ fp_closed = getattr(self._fp, "closed", False)
+
+ with self._error_catcher():
+ data = self._fp_read(amt, read1=read1) if not fp_closed else b""
+ if amt is not None and amt != 0 and not data:
+ # Platform-specific: Buggy versions of Python.
+ # Close the connection when no data is returned
+ #
+ # This is redundant to what httplib/http.client _should_
+ # already do. However, versions of python released before
+ # December 15, 2012 (http://bugs.python.org/issue16298) do
+ # not properly close the connection in all cases. There is
+ # no harm in redundantly calling close.
+ self._fp.close()
+ if (
+ self.enforce_content_length
+ and self.length_remaining is not None
+ and self.length_remaining != 0
+ ):
+ # This is an edge case that httplib failed to cover due
+ # to concerns of backward compatibility. We're
+ # addressing it here to make sure IncompleteRead is
+ # raised during streaming, so all calls with incorrect
+ # Content-Length are caught.
+ raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
+ elif read1 and (
+ (amt != 0 and not data) or self.length_remaining == len(data)
+ ):
+ # All data has been read, but `self._fp.read1` in
+ # CPython 3.12 and older doesn't always close
+ # `http.client.HTTPResponse`, so we close it here.
+ # See https://github.com/python/cpython/issues/113199
+ self._fp.close()
+
+ if data:
+ self._fp_bytes_read += len(data)
+ if self.length_remaining is not None:
+ self.length_remaining -= len(data)
+ return data
+
+ def read(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ cache_content: bool = False,
+ ) -> bytes:
+ """
+ Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
+ parameters: ``decode_content`` and ``cache_content``.
+
+ :param amt:
+ How much of the content to read. If specified, caching is skipped
+ because it doesn't make sense to cache partial content as the full
+ response.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+
+ :param cache_content:
+ If True, will save the returned data such that the same result is
+ returned despite of the state of the underlying file object. This
+ is useful if you want the ``.data`` property to continue working
+ after having ``.read()`` the file object. (Overridden if ``amt`` is
+ set.)
+ """
+ self._init_decoder()
+ if decode_content is None:
+ decode_content = self.decode_content
+
+ if amt and amt < 0:
+ # Negative numbers and `None` should be treated the same.
+ amt = None
+ elif amt is not None:
+ cache_content = False
+
+ if len(self._decoded_buffer) >= amt:
+ return self._decoded_buffer.get(amt)
+
+ data = self._raw_read(amt)
+
+ flush_decoder = amt is None or (amt != 0 and not data)
+
+ if not data and len(self._decoded_buffer) == 0:
+ return data
+
+ if amt is None:
+ data = self._decode(data, decode_content, flush_decoder)
+ if cache_content:
+ self._body = data
+ else:
+ # do not waste memory on buffer when not decoding
+ if not decode_content:
+ if self._has_decoded_content:
+ raise RuntimeError(
+ "Calling read(decode_content=False) is not supported after "
+ "read(decode_content=True) was called."
+ )
+ return data
+
+ decoded_data = self._decode(data, decode_content, flush_decoder)
+ self._decoded_buffer.put(decoded_data)
+
+ while len(self._decoded_buffer) < amt and data:
+ # TODO make sure to initially read enough data to get past the headers
+ # For example, the GZ file header takes 10 bytes, we don't want to read
+ # it one byte at a time
+ data = self._raw_read(amt)
+ decoded_data = self._decode(data, decode_content, flush_decoder)
+ self._decoded_buffer.put(decoded_data)
+ data = self._decoded_buffer.get(amt)
+
+ return data
+
+ def read1(
+ self,
+ amt: int | None = None,
+ decode_content: bool | None = None,
+ ) -> bytes:
+ """
+ Similar to ``http.client.HTTPResponse.read1`` and documented
+ in :meth:`io.BufferedReader.read1`, but with an additional parameter:
+ ``decode_content``.
+
+ :param amt:
+ How much of the content to read.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ if decode_content is None:
+ decode_content = self.decode_content
+ if amt and amt < 0:
+ # Negative numbers and `None` should be treated the same.
+ amt = None
+ # try and respond without going to the network
+ if self._has_decoded_content:
+ if not decode_content:
+ raise RuntimeError(
+ "Calling read1(decode_content=False) is not supported after "
+ "read1(decode_content=True) was called."
+ )
+ if len(self._decoded_buffer) > 0:
+ if amt is None:
+ return self._decoded_buffer.get_all()
+ return self._decoded_buffer.get(amt)
+ if amt == 0:
+ return b""
+
+ # FIXME, this method's type doesn't say returning None is possible
+ data = self._raw_read(amt, read1=True)
+ if not decode_content or data is None:
+ return data
+
+ self._init_decoder()
+ while True:
+ flush_decoder = not data
+ decoded_data = self._decode(data, decode_content, flush_decoder)
+ self._decoded_buffer.put(decoded_data)
+ if decoded_data or flush_decoder:
+ break
+ data = self._raw_read(8192, read1=True)
+
+ if amt is None:
+ return self._decoded_buffer.get_all()
+ return self._decoded_buffer.get(amt)
+
+ def stream(
+ self, amt: int | None = 2**16, decode_content: bool | None = None
+ ) -> typing.Generator[bytes]:
+ """
+ A generator wrapper for the read() method. A call will block until
+ ``amt`` bytes have been read from the connection or until the
+ connection is closed.
+
+ :param amt:
+ How much of the content to read. The generator will return up to
+ much data per iteration, but may return less. This is particularly
+ likely when using compressed data. However, the empty string will
+ never be returned.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ if self.chunked and self.supports_chunked_reads():
+ yield from self.read_chunked(amt, decode_content=decode_content)
+ else:
+ while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0:
+ data = self.read(amt=amt, decode_content=decode_content)
+
+ if data:
+ yield data
+
+ # Overrides from io.IOBase
+ def readable(self) -> bool:
+ return True
+
+ def shutdown(self) -> None:
+ if not self._sock_shutdown:
+ raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set")
+ if self._connection is None:
+ raise RuntimeError(
+ "Cannot shutdown as connection has already been released to the pool"
+ )
+ self._sock_shutdown(socket.SHUT_RD)
+
+ def close(self) -> None:
+ self._sock_shutdown = None
+
+ if not self.closed and self._fp:
+ self._fp.close()
+
+ if self._connection:
+ self._connection.close()
+
+ if not self.auto_close:
+ io.IOBase.close(self)
+
+ @property
+ def closed(self) -> bool:
+ if not self.auto_close:
+ return io.IOBase.closed.__get__(self) # type: ignore[no-any-return]
+ elif self._fp is None:
+ return True
+ elif hasattr(self._fp, "isclosed"):
+ return self._fp.isclosed()
+ elif hasattr(self._fp, "closed"):
+ return self._fp.closed
+ else:
+ return True
+
+ def fileno(self) -> int:
+ if self._fp is None:
+ raise OSError("HTTPResponse has no file to get a fileno from")
+ elif hasattr(self._fp, "fileno"):
+ return self._fp.fileno()
+ else:
+ raise OSError(
+ "The file-like object this HTTPResponse is wrapped "
+ "around has no file descriptor"
+ )
+
+ def flush(self) -> None:
+ if (
+ self._fp is not None
+ and hasattr(self._fp, "flush")
+ and not getattr(self._fp, "closed", False)
+ ):
+ return self._fp.flush()
+
+ def supports_chunked_reads(self) -> bool:
+ """
+ Checks if the underlying file-like object looks like a
+ :class:`http.client.HTTPResponse` object. We do this by testing for
+ the fp attribute. If it is present we assume it returns raw chunks as
+ processed by read_chunked().
+ """
+ return hasattr(self._fp, "fp")
+
+ def _update_chunk_length(self) -> None:
+ # First, we'll figure out length of a chunk and then
+ # we'll try to read it from socket.
+ if self.chunk_left is not None:
+ return None
+ line = self._fp.fp.readline() # type: ignore[union-attr]
+ line = line.split(b";", 1)[0]
+ try:
+ self.chunk_left = int(line, 16)
+ except ValueError:
+ self.close()
+ if line:
+ # Invalid chunked protocol response, abort.
+ raise InvalidChunkLength(self, line) from None
+ else:
+ # Truncated at start of next chunk
+ raise ProtocolError("Response ended prematurely") from None
+
+ def _handle_chunk(self, amt: int | None) -> bytes:
+ returned_chunk = None
+ if amt is None:
+ chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
+ returned_chunk = chunk
+ self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ elif self.chunk_left is not None and amt < self.chunk_left:
+ value = self._fp._safe_read(amt) # type: ignore[union-attr]
+ self.chunk_left = self.chunk_left - amt
+ returned_chunk = value
+ elif amt == self.chunk_left:
+ value = self._fp._safe_read(amt) # type: ignore[union-attr]
+ self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ returned_chunk = value
+ else: # amt > self.chunk_left
+ returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
+ self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ return returned_chunk # type: ignore[no-any-return]
+
+ def read_chunked(
+ self, amt: int | None = None, decode_content: bool | None = None
+ ) -> typing.Generator[bytes]:
+ """
+ Similar to :meth:`HTTPResponse.read`, but with an additional
+ parameter: ``decode_content``.
+
+ :param amt:
+ How much of the content to read. If specified, caching is skipped
+ because it doesn't make sense to cache partial content as the full
+ response.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ self._init_decoder()
+ # FIXME: Rewrite this method and make it a class with a better structured logic.
+ if not self.chunked:
+ raise ResponseNotChunked(
+ "Response is not chunked. "
+ "Header 'transfer-encoding: chunked' is missing."
+ )
+ if not self.supports_chunked_reads():
+ raise BodyNotHttplibCompatible(
+ "Body should be http.client.HTTPResponse like. "
+ "It should have have an fp attribute which returns raw chunks."
+ )
+
+ with self._error_catcher():
+ # Don't bother reading the body of a HEAD request.
+ if self._original_response and is_response_to_head(self._original_response):
+ self._original_response.close()
+ return None
+
+ # If a response is already read and closed
+ # then return immediately.
+ if self._fp.fp is None: # type: ignore[union-attr]
+ return None
+
+ if amt and amt < 0:
+ # Negative numbers and `None` should be treated the same,
+ # but httplib handles only `None` correctly.
+ amt = None
+
+ while True:
+ self._update_chunk_length()
+ if self.chunk_left == 0:
+ break
+ chunk = self._handle_chunk(amt)
+ decoded = self._decode(
+ chunk, decode_content=decode_content, flush_decoder=False
+ )
+ if decoded:
+ yield decoded
+
+ if decode_content:
+ # On CPython and PyPy, we should never need to flush the
+ # decoder. However, on Jython we *might* need to, so
+ # lets defensively do it anyway.
+ decoded = self._flush_decoder()
+ if decoded: # Platform-specific: Jython.
+ yield decoded
+
+ # Chunk content ends with \r\n: discard it.
+ while self._fp is not None:
+ line = self._fp.fp.readline()
+ if not line:
+ # Some sites may not end with '\r\n'.
+ break
+ if line == b"\r\n":
+ break
+
+ # We read everything; close the "file".
+ if self._original_response:
+ self._original_response.close()
+
+ @property
+ def url(self) -> str | None:
+ """
+ Returns the URL that was the source of this response.
+ If the request that generated this response redirected, this method
+ will return the final redirect location.
+ """
+ return self._request_url
+
+ @url.setter
+ def url(self, url: str) -> None:
+ self._request_url = url
+
+ def __iter__(self) -> typing.Iterator[bytes]:
+ buffer: list[bytes] = []
+ for chunk in self.stream(decode_content=True):
+ if b"\n" in chunk:
+ chunks = chunk.split(b"\n")
+ yield b"".join(buffer) + chunks[0] + b"\n"
+ for x in chunks[1:-1]:
+ yield x + b"\n"
+ if chunks[-1]:
+ buffer = [chunks[-1]]
+ else:
+ buffer = []
+ else:
+ buffer.append(chunk)
+ if buffer:
+ yield b"".join(buffer)
diff --git a/phivenv/Lib/site-packages/urllib3/util/__init__.py b/phivenv/Lib/site-packages/urllib3/util/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..534126033c083203649022fa9b753a433f005556
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/__init__.py
@@ -0,0 +1,42 @@
+# For backwards compatibility, provide imports that used to be here.
+from __future__ import annotations
+
+from .connection import is_connection_dropped
+from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
+from .response import is_fp_closed
+from .retry import Retry
+from .ssl_ import (
+ ALPN_PROTOCOLS,
+ IS_PYOPENSSL,
+ SSLContext,
+ assert_fingerprint,
+ create_urllib3_context,
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ ssl_wrap_socket,
+)
+from .timeout import Timeout
+from .url import Url, parse_url
+from .wait import wait_for_read, wait_for_write
+
+__all__ = (
+ "IS_PYOPENSSL",
+ "SSLContext",
+ "ALPN_PROTOCOLS",
+ "Retry",
+ "Timeout",
+ "Url",
+ "assert_fingerprint",
+ "create_urllib3_context",
+ "is_connection_dropped",
+ "is_fp_closed",
+ "parse_url",
+ "make_headers",
+ "resolve_cert_reqs",
+ "resolve_ssl_version",
+ "ssl_wrap_socket",
+ "wait_for_read",
+ "wait_for_write",
+ "SKIP_HEADER",
+ "SKIPPABLE_HEADERS",
+)
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5c04059f26616cac14e29d813fef0f8856fcbe53
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/connection.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/connection.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e191bbe8e08349f1aa44a3e6cf6d263979ee23ec
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/connection.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/proxy.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/proxy.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..09076bfa7afb0e63db39a8cac3ceb0990205552f
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/proxy.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/request.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/request.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..074cfbe22aebe0e83b65b09f2685014ffcd4140e
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/request.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/response.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/response.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..eac2fe19e952e904f5d128c008a07f053544bd00
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/response.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/retry.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/retry.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..47cb4ec06a90392cc9955963fa4adea11c339b2e
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/retry.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bb3059291f2386c9f7f29452ec15f3c31a8e72ce
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_match_hostname.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_match_hostname.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4fd0d680f7035a8c2d36e094c15a3c76915ef127
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssl_match_hostname.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssltransport.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssltransport.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..55ad73e2b106182641b74fe29ed5595fa3301e10
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/ssltransport.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/timeout.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/timeout.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..178bb60193c273b298563f1b1ca88825af052525
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/timeout.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/url.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/url.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ca10864887f1ae8ed00e0686ace36a956cde1eca
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/url.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/util.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/util.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f3bbd1294cefad6b946a714234fa1505edc476c
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/util.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/__pycache__/wait.cpython-39.pyc b/phivenv/Lib/site-packages/urllib3/util/__pycache__/wait.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ceef1f64f009636c56c2c606c02b942ec975fe11
Binary files /dev/null and b/phivenv/Lib/site-packages/urllib3/util/__pycache__/wait.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/urllib3/util/connection.py b/phivenv/Lib/site-packages/urllib3/util/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..f92519ee9124e91e5da7d60ccc3f274312ed3514
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/connection.py
@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import socket
+import typing
+
+from ..exceptions import LocationParseError
+from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
+
+_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]]
+
+if typing.TYPE_CHECKING:
+ from .._base_connection import BaseHTTPConnection
+
+
+def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific
+ """
+ Returns True if the connection is dropped and should be closed.
+ :param conn: :class:`urllib3.connection.HTTPConnection` object.
+ """
+ return not conn.is_connected
+
+
+# This function is copied from socket.py in the Python 2.7 standard
+# library test suite. Added to its signature is only `socket_options`.
+# One additional modification is that we avoid binding to IPv6 servers
+# discovered in DNS if the system doesn't have IPv6 functionality.
+def create_connection(
+ address: tuple[str, int],
+ timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ source_address: tuple[str, int] | None = None,
+ socket_options: _TYPE_SOCKET_OPTIONS | None = None,
+) -> socket.socket:
+ """Connect to *address* and return the socket object.
+
+ Convenience function. Connect to *address* (a 2-tuple ``(host,
+ port)``) and return the socket object. Passing the optional
+ *timeout* parameter will set the timeout on the socket instance
+ before attempting to connect. If no *timeout* is supplied, the
+ global default timeout setting returned by :func:`socket.getdefaulttimeout`
+ is used. If *source_address* is set it must be a tuple of (host, port)
+ for the socket to bind as a source address before making the connection.
+ An host of '' or port 0 tells the OS to use the default.
+ """
+
+ host, port = address
+ if host.startswith("["):
+ host = host.strip("[]")
+ err = None
+
+ # Using the value from allowed_gai_family() in the context of getaddrinfo lets
+ # us select whether to work with IPv4 DNS records, IPv6 records, or both.
+ # The original create_connection function always returns all records.
+ family = allowed_gai_family()
+
+ try:
+ host.encode("idna")
+ except UnicodeError:
+ raise LocationParseError(f"'{host}', label empty or too long") from None
+
+ for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
+ af, socktype, proto, canonname, sa = res
+ sock = None
+ try:
+ sock = socket.socket(af, socktype, proto)
+
+ # If provided, set socket level options before connecting.
+ _set_socket_options(sock, socket_options)
+
+ if timeout is not _DEFAULT_TIMEOUT:
+ sock.settimeout(timeout)
+ if source_address:
+ sock.bind(source_address)
+ sock.connect(sa)
+ # Break explicitly a reference cycle
+ err = None
+ return sock
+
+ except OSError as _:
+ err = _
+ if sock is not None:
+ sock.close()
+
+ if err is not None:
+ try:
+ raise err
+ finally:
+ # Break explicitly a reference cycle
+ err = None
+ else:
+ raise OSError("getaddrinfo returns an empty list")
+
+
+def _set_socket_options(
+ sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None
+) -> None:
+ if options is None:
+ return
+
+ for opt in options:
+ sock.setsockopt(*opt)
+
+
+def allowed_gai_family() -> socket.AddressFamily:
+ """This function is designed to work in the context of
+ getaddrinfo, where family=socket.AF_UNSPEC is the default and
+ will perform a DNS search for both IPv6 and IPv4 records."""
+
+ family = socket.AF_INET
+ if HAS_IPV6:
+ family = socket.AF_UNSPEC
+ return family
+
+
+def _has_ipv6(host: str) -> bool:
+ """Returns True if the system can bind an IPv6 address."""
+ sock = None
+ has_ipv6 = False
+
+ if socket.has_ipv6:
+ # has_ipv6 returns true if cPython was compiled with IPv6 support.
+ # It does not tell us if the system has IPv6 support enabled. To
+ # determine that we must bind to an IPv6 address.
+ # https://github.com/urllib3/urllib3/pull/611
+ # https://bugs.python.org/issue658327
+ try:
+ sock = socket.socket(socket.AF_INET6)
+ sock.bind((host, 0))
+ has_ipv6 = True
+ except Exception:
+ pass
+
+ if sock:
+ sock.close()
+ return has_ipv6
+
+
+HAS_IPV6 = _has_ipv6("::1")
diff --git a/phivenv/Lib/site-packages/urllib3/util/proxy.py b/phivenv/Lib/site-packages/urllib3/util/proxy.py
new file mode 100644
index 0000000000000000000000000000000000000000..908fc6621d0afbed16bde2c1957a5cf28d3a84d8
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/proxy.py
@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import typing
+
+from .url import Url
+
+if typing.TYPE_CHECKING:
+ from ..connection import ProxyConfig
+
+
+def connection_requires_http_tunnel(
+ proxy_url: Url | None = None,
+ proxy_config: ProxyConfig | None = None,
+ destination_scheme: str | None = None,
+) -> bool:
+ """
+ Returns True if the connection requires an HTTP CONNECT through the proxy.
+
+ :param URL proxy_url:
+ URL of the proxy.
+ :param ProxyConfig proxy_config:
+ Proxy configuration from poolmanager.py
+ :param str destination_scheme:
+ The scheme of the destination. (i.e https, http, etc)
+ """
+ # If we're not using a proxy, no way to use a tunnel.
+ if proxy_url is None:
+ return False
+
+ # HTTP destinations never require tunneling, we always forward.
+ if destination_scheme == "http":
+ return False
+
+ # Support for forwarding with HTTPS proxies and HTTPS destinations.
+ if (
+ proxy_url.scheme == "https"
+ and proxy_config
+ and proxy_config.use_forwarding_for_https
+ ):
+ return False
+
+ # Otherwise always use a tunnel.
+ return True
diff --git a/phivenv/Lib/site-packages/urllib3/util/request.py b/phivenv/Lib/site-packages/urllib3/util/request.py
new file mode 100644
index 0000000000000000000000000000000000000000..23605c522b85f940f3af4db2e0561c48b0b0d269
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/request.py
@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import io
+import typing
+from base64 import b64encode
+from enum import Enum
+
+from ..exceptions import UnrewindableBodyError
+from .util import to_bytes
+
+if typing.TYPE_CHECKING:
+ from typing import Final
+
+# Pass as a value within ``headers`` to skip
+# emitting some HTTP headers that are added automatically.
+# The only headers that are supported are ``Accept-Encoding``,
+# ``Host``, and ``User-Agent``.
+SKIP_HEADER = "@@@SKIP_HEADER@@@"
+SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
+
+ACCEPT_ENCODING = "gzip,deflate"
+try:
+ try:
+ import brotlicffi as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
+ except ImportError:
+ import brotli as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
+except ImportError:
+ pass
+else:
+ ACCEPT_ENCODING += ",br"
+
+try:
+ from compression import ( # type: ignore[import-not-found] # noqa: F401
+ zstd as _unused_module_zstd,
+ )
+
+ ACCEPT_ENCODING += ",zstd"
+except ImportError:
+ try:
+ import zstandard as _unused_module_zstd # noqa: F401
+
+ ACCEPT_ENCODING += ",zstd"
+ except ImportError:
+ pass
+
+
+class _TYPE_FAILEDTELL(Enum):
+ token = 0
+
+
+_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token
+
+_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL]
+
+# When sending a request with these methods we aren't expecting
+# a body so don't need to set an explicit 'Content-Length: 0'
+# The reason we do this in the negative instead of tracking methods
+# which 'should' have a body is because unknown methods should be
+# treated as if they were 'POST' which *does* expect a body.
+_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"}
+
+
+def make_headers(
+ keep_alive: bool | None = None,
+ accept_encoding: bool | list[str] | str | None = None,
+ user_agent: str | None = None,
+ basic_auth: str | None = None,
+ proxy_basic_auth: str | None = None,
+ disable_cache: bool | None = None,
+) -> dict[str, str]:
+ """
+ Shortcuts for generating request headers.
+
+ :param keep_alive:
+ If ``True``, adds 'connection: keep-alive' header.
+
+ :param accept_encoding:
+ Can be a boolean, list, or string.
+ ``True`` translates to 'gzip,deflate'. If the dependencies for
+ Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or Zstandard
+ (the ``zstandard`` package) algorithms are installed, then their encodings are
+ included in the string ('br' and 'zstd', respectively).
+ List will get joined by comma.
+ String will be used as provided.
+
+ :param user_agent:
+ String representing the user-agent you want, such as
+ "python-urllib3/0.6"
+
+ :param basic_auth:
+ Colon-separated username:password string for 'authorization: basic ...'
+ auth header.
+
+ :param proxy_basic_auth:
+ Colon-separated username:password string for 'proxy-authorization: basic ...'
+ auth header.
+
+ :param disable_cache:
+ If ``True``, adds 'cache-control: no-cache' header.
+
+ Example:
+
+ .. code-block:: python
+
+ import urllib3
+
+ print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0"))
+ # {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
+ print(urllib3.util.make_headers(accept_encoding=True))
+ # {'accept-encoding': 'gzip,deflate'}
+ """
+ headers: dict[str, str] = {}
+ if accept_encoding:
+ if isinstance(accept_encoding, str):
+ pass
+ elif isinstance(accept_encoding, list):
+ accept_encoding = ",".join(accept_encoding)
+ else:
+ accept_encoding = ACCEPT_ENCODING
+ headers["accept-encoding"] = accept_encoding
+
+ if user_agent:
+ headers["user-agent"] = user_agent
+
+ if keep_alive:
+ headers["connection"] = "keep-alive"
+
+ if basic_auth:
+ headers["authorization"] = (
+ f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}"
+ )
+
+ if proxy_basic_auth:
+ headers["proxy-authorization"] = (
+ f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}"
+ )
+
+ if disable_cache:
+ headers["cache-control"] = "no-cache"
+
+ return headers
+
+
+def set_file_position(
+ body: typing.Any, pos: _TYPE_BODY_POSITION | None
+) -> _TYPE_BODY_POSITION | None:
+ """
+ If a position is provided, move file to that point.
+ Otherwise, we'll attempt to record a position for future use.
+ """
+ if pos is not None:
+ rewind_body(body, pos)
+ elif getattr(body, "tell", None) is not None:
+ try:
+ pos = body.tell()
+ except OSError:
+ # This differentiates from None, allowing us to catch
+ # a failed `tell()` later when trying to rewind the body.
+ pos = _FAILEDTELL
+
+ return pos
+
+
+def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None:
+ """
+ Attempt to rewind body to a certain position.
+ Primarily used for request redirects and retries.
+
+ :param body:
+ File-like object that supports seek.
+
+ :param int pos:
+ Position to seek to in file.
+ """
+ body_seek = getattr(body, "seek", None)
+ if body_seek is not None and isinstance(body_pos, int):
+ try:
+ body_seek(body_pos)
+ except OSError as e:
+ raise UnrewindableBodyError(
+ "An error occurred when rewinding request body for redirect/retry."
+ ) from e
+ elif body_pos is _FAILEDTELL:
+ raise UnrewindableBodyError(
+ "Unable to record file position for rewinding "
+ "request body during a redirect/retry."
+ )
+ else:
+ raise ValueError(
+ f"body_pos must be of type integer, instead it was {type(body_pos)}."
+ )
+
+
+class ChunksAndContentLength(typing.NamedTuple):
+ chunks: typing.Iterable[bytes] | None
+ content_length: int | None
+
+
+def body_to_chunks(
+ body: typing.Any | None, method: str, blocksize: int
+) -> ChunksAndContentLength:
+ """Takes the HTTP request method, body, and blocksize and
+ transforms them into an iterable of chunks to pass to
+ socket.sendall() and an optional 'Content-Length' header.
+
+ A 'Content-Length' of 'None' indicates the length of the body
+ can't be determined so should use 'Transfer-Encoding: chunked'
+ for framing instead.
+ """
+
+ chunks: typing.Iterable[bytes] | None
+ content_length: int | None
+
+ # No body, we need to make a recommendation on 'Content-Length'
+ # based on whether that request method is expected to have
+ # a body or not.
+ if body is None:
+ chunks = None
+ if method.upper() not in _METHODS_NOT_EXPECTING_BODY:
+ content_length = 0
+ else:
+ content_length = None
+
+ # Bytes or strings become bytes
+ elif isinstance(body, (str, bytes)):
+ chunks = (to_bytes(body),)
+ content_length = len(chunks[0])
+
+ # File-like object, TODO: use seek() and tell() for length?
+ elif hasattr(body, "read"):
+
+ def chunk_readable() -> typing.Iterable[bytes]:
+ nonlocal body, blocksize
+ encode = isinstance(body, io.TextIOBase)
+ while True:
+ datablock = body.read(blocksize)
+ if not datablock:
+ break
+ if encode:
+ datablock = datablock.encode("utf-8")
+ yield datablock
+
+ chunks = chunk_readable()
+ content_length = None
+
+ # Otherwise we need to start checking via duck-typing.
+ else:
+ try:
+ # Check if the body implements the buffer API.
+ mv = memoryview(body)
+ except TypeError:
+ try:
+ # Check if the body is an iterable
+ chunks = iter(body)
+ content_length = None
+ except TypeError:
+ raise TypeError(
+ f"'body' must be a bytes-like object, file-like "
+ f"object, or iterable. Instead was {body!r}"
+ ) from None
+ else:
+ # Since it implements the buffer API can be passed directly to socket.sendall()
+ chunks = (body,)
+ content_length = mv.nbytes
+
+ return ChunksAndContentLength(chunks=chunks, content_length=content_length)
diff --git a/phivenv/Lib/site-packages/urllib3/util/response.py b/phivenv/Lib/site-packages/urllib3/util/response.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f4578696fa2e17a900c6890ec26d65e860b0b72
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/response.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import http.client as httplib
+from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
+
+from ..exceptions import HeaderParsingError
+
+
+def is_fp_closed(obj: object) -> bool:
+ """
+ Checks whether a given file-like object is closed.
+
+ :param obj:
+ The file-like object to check.
+ """
+
+ try:
+ # Check `isclosed()` first, in case Python3 doesn't set `closed`.
+ # GH Issue #928
+ return obj.isclosed() # type: ignore[no-any-return, attr-defined]
+ except AttributeError:
+ pass
+
+ try:
+ # Check via the official file-like-object way.
+ return obj.closed # type: ignore[no-any-return, attr-defined]
+ except AttributeError:
+ pass
+
+ try:
+ # Check if the object is a container for another file-like object that
+ # gets released on exhaustion (e.g. HTTPResponse).
+ return obj.fp is None # type: ignore[attr-defined]
+ except AttributeError:
+ pass
+
+ raise ValueError("Unable to determine whether fp is closed.")
+
+
+def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
+ """
+ Asserts whether all headers have been successfully parsed.
+ Extracts encountered errors from the result of parsing headers.
+
+ Only works on Python 3.
+
+ :param http.client.HTTPMessage headers: Headers to verify.
+
+ :raises urllib3.exceptions.HeaderParsingError:
+ If parsing errors are found.
+ """
+
+ # This will fail silently if we pass in the wrong kind of parameter.
+ # To make debugging easier add an explicit check.
+ if not isinstance(headers, httplib.HTTPMessage):
+ raise TypeError(f"expected httplib.Message, got {type(headers)}.")
+
+ unparsed_data = None
+
+ # get_payload is actually email.message.Message.get_payload;
+ # we're only interested in the result if it's not a multipart message
+ if not headers.is_multipart():
+ payload = headers.get_payload()
+
+ if isinstance(payload, (bytes, str)):
+ unparsed_data = payload
+
+ # httplib is assuming a response body is available
+ # when parsing headers even when httplib only sends
+ # header data to parse_headers() This results in
+ # defects on multipart responses in particular.
+ # See: https://github.com/urllib3/urllib3/issues/800
+
+ # So we ignore the following defects:
+ # - StartBoundaryNotFoundDefect:
+ # The claimed start boundary was never found.
+ # - MultipartInvariantViolationDefect:
+ # A message claimed to be a multipart but no subparts were found.
+ defects = [
+ defect
+ for defect in headers.defects
+ if not isinstance(
+ defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
+ )
+ ]
+
+ if defects or unparsed_data:
+ raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
+
+
+def is_response_to_head(response: httplib.HTTPResponse) -> bool:
+ """
+ Checks whether the request of a response has been a HEAD-request.
+
+ :param http.client.HTTPResponse response:
+ Response to check if the originating request
+ used 'HEAD' as a method.
+ """
+ # FIXME: Can we do this somehow without accessing private httplib _method?
+ method_str = response._method # type: str # type: ignore[attr-defined]
+ return method_str.upper() == "HEAD"
diff --git a/phivenv/Lib/site-packages/urllib3/util/retry.py b/phivenv/Lib/site-packages/urllib3/util/retry.py
new file mode 100644
index 0000000000000000000000000000000000000000..0456cceba47b16ae6784458cc17eaa528a517ffa
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/retry.py
@@ -0,0 +1,533 @@
+from __future__ import annotations
+
+import email
+import logging
+import random
+import re
+import time
+import typing
+from itertools import takewhile
+from types import TracebackType
+
+from ..exceptions import (
+ ConnectTimeoutError,
+ InvalidHeader,
+ MaxRetryError,
+ ProtocolError,
+ ProxyError,
+ ReadTimeoutError,
+ ResponseError,
+)
+from .util import reraise
+
+if typing.TYPE_CHECKING:
+ from typing_extensions import Self
+
+ from ..connectionpool import ConnectionPool
+ from ..response import BaseHTTPResponse
+
+log = logging.getLogger(__name__)
+
+
+# Data structure for representing the metadata of requests that result in a retry.
+class RequestHistory(typing.NamedTuple):
+ method: str | None
+ url: str | None
+ error: Exception | None
+ status: int | None
+ redirect_location: str | None
+
+
+class Retry:
+ """Retry configuration.
+
+ Each retry attempt will create a new Retry object with updated values, so
+ they can be safely reused.
+
+ Retries can be defined as a default for a pool:
+
+ .. code-block:: python
+
+ retries = Retry(connect=5, read=2, redirect=5)
+ http = PoolManager(retries=retries)
+ response = http.request("GET", "https://example.com/")
+
+ Or per-request (which overrides the default for the pool):
+
+ .. code-block:: python
+
+ response = http.request("GET", "https://example.com/", retries=Retry(10))
+
+ Retries can be disabled by passing ``False``:
+
+ .. code-block:: python
+
+ response = http.request("GET", "https://example.com/", retries=False)
+
+ Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
+ retries are disabled, in which case the causing exception will be raised.
+
+ :param int total:
+ Total number of retries to allow. Takes precedence over other counts.
+
+ Set to ``None`` to remove this constraint and fall back on other
+ counts.
+
+ Set to ``0`` to fail on the first retry.
+
+ Set to ``False`` to disable and imply ``raise_on_redirect=False``.
+
+ :param int connect:
+ How many connection-related errors to retry on.
+
+ These are errors raised before the request is sent to the remote server,
+ which we assume has not triggered the server to process the request.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param int read:
+ How many times to retry on read errors.
+
+ These errors are raised after the request was sent to the server, so the
+ request may have side-effects.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param int redirect:
+ How many redirects to perform. Limit this to avoid infinite redirect
+ loops.
+
+ A redirect is a HTTP response with a status code 301, 302, 303, 307 or
+ 308.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ Set to ``False`` to disable and imply ``raise_on_redirect=False``.
+
+ :param int status:
+ How many times to retry on bad status codes.
+
+ These are retries made on responses, where status code matches
+ ``status_forcelist``.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ :param int other:
+ How many times to retry on other errors.
+
+ Other errors are errors that are not connect, read, redirect or status errors.
+ These errors might be raised after the request was sent to the server, so the
+ request might have side-effects.
+
+ Set to ``0`` to fail on the first retry of this type.
+
+ If ``total`` is not set, it's a good idea to set this to 0 to account
+ for unexpected edge cases and avoid infinite retry loops.
+
+ :param Collection allowed_methods:
+ Set of uppercased HTTP method verbs that we should retry on.
+
+ By default, we only retry on methods which are considered to be
+ idempotent (multiple requests with the same parameters end with the
+ same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
+
+ Set to a ``None`` value to retry on any verb.
+
+ :param Collection status_forcelist:
+ A set of integer HTTP status codes that we should force a retry on.
+ A retry is initiated if the request method is in ``allowed_methods``
+ and the response status code is in ``status_forcelist``.
+
+ By default, this is disabled with ``None``.
+
+ :param float backoff_factor:
+ A backoff factor to apply between attempts after the second try
+ (most errors are resolved immediately by a second try without a
+ delay). urllib3 will sleep for::
+
+ {backoff factor} * (2 ** ({number of previous retries}))
+
+ seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
+
+ random.uniform(0, {backoff jitter})
+
+ seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
+ sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
+ be longer than `backoff_max`.
+
+ By default, backoff is disabled (factor set to 0).
+
+ :param bool raise_on_redirect: Whether, if the number of redirects is
+ exhausted, to raise a MaxRetryError, or to return a response with a
+ response code in the 3xx range.
+
+ :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
+ whether we should raise an exception, or return a response,
+ if status falls in ``status_forcelist`` range and retries have
+ been exhausted.
+
+ :param tuple history: The history of the request encountered during
+ each call to :meth:`~Retry.increment`. The list is in the order
+ the requests occurred. Each list item is of class :class:`RequestHistory`.
+
+ :param bool respect_retry_after_header:
+ Whether to respect Retry-After header on status codes defined as
+ :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
+
+ :param Collection remove_headers_on_redirect:
+ Sequence of headers to remove from the request when a response
+ indicating a redirect is returned before firing off the redirected
+ request.
+ """
+
+ #: Default methods to be used for ``allowed_methods``
+ DEFAULT_ALLOWED_METHODS = frozenset(
+ ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
+ )
+
+ #: Default status codes to be used for ``status_forcelist``
+ RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
+
+ #: Default headers to be used for ``remove_headers_on_redirect``
+ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(
+ ["Cookie", "Authorization", "Proxy-Authorization"]
+ )
+
+ #: Default maximum backoff time.
+ DEFAULT_BACKOFF_MAX = 120
+
+ # Backward compatibility; assigned outside of the class.
+ DEFAULT: typing.ClassVar[Retry]
+
+ def __init__(
+ self,
+ total: bool | int | None = 10,
+ connect: int | None = None,
+ read: int | None = None,
+ redirect: bool | int | None = None,
+ status: int | None = None,
+ other: int | None = None,
+ allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
+ status_forcelist: typing.Collection[int] | None = None,
+ backoff_factor: float = 0,
+ backoff_max: float = DEFAULT_BACKOFF_MAX,
+ raise_on_redirect: bool = True,
+ raise_on_status: bool = True,
+ history: tuple[RequestHistory, ...] | None = None,
+ respect_retry_after_header: bool = True,
+ remove_headers_on_redirect: typing.Collection[
+ str
+ ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
+ backoff_jitter: float = 0.0,
+ ) -> None:
+ self.total = total
+ self.connect = connect
+ self.read = read
+ self.status = status
+ self.other = other
+
+ if redirect is False or total is False:
+ redirect = 0
+ raise_on_redirect = False
+
+ self.redirect = redirect
+ self.status_forcelist = status_forcelist or set()
+ self.allowed_methods = allowed_methods
+ self.backoff_factor = backoff_factor
+ self.backoff_max = backoff_max
+ self.raise_on_redirect = raise_on_redirect
+ self.raise_on_status = raise_on_status
+ self.history = history or ()
+ self.respect_retry_after_header = respect_retry_after_header
+ self.remove_headers_on_redirect = frozenset(
+ h.lower() for h in remove_headers_on_redirect
+ )
+ self.backoff_jitter = backoff_jitter
+
+ def new(self, **kw: typing.Any) -> Self:
+ params = dict(
+ total=self.total,
+ connect=self.connect,
+ read=self.read,
+ redirect=self.redirect,
+ status=self.status,
+ other=self.other,
+ allowed_methods=self.allowed_methods,
+ status_forcelist=self.status_forcelist,
+ backoff_factor=self.backoff_factor,
+ backoff_max=self.backoff_max,
+ raise_on_redirect=self.raise_on_redirect,
+ raise_on_status=self.raise_on_status,
+ history=self.history,
+ remove_headers_on_redirect=self.remove_headers_on_redirect,
+ respect_retry_after_header=self.respect_retry_after_header,
+ backoff_jitter=self.backoff_jitter,
+ )
+
+ params.update(kw)
+ return type(self)(**params) # type: ignore[arg-type]
+
+ @classmethod
+ def from_int(
+ cls,
+ retries: Retry | bool | int | None,
+ redirect: bool | int | None = True,
+ default: Retry | bool | int | None = None,
+ ) -> Retry:
+ """Backwards-compatibility for the old retries format."""
+ if retries is None:
+ retries = default if default is not None else cls.DEFAULT
+
+ if isinstance(retries, Retry):
+ return retries
+
+ redirect = bool(redirect) and None
+ new_retries = cls(retries, redirect=redirect)
+ log.debug("Converted retries value: %r -> %r", retries, new_retries)
+ return new_retries
+
+ def get_backoff_time(self) -> float:
+ """Formula for computing the current backoff
+
+ :rtype: float
+ """
+ # We want to consider only the last consecutive errors sequence (Ignore redirects).
+ consecutive_errors_len = len(
+ list(
+ takewhile(lambda x: x.redirect_location is None, reversed(self.history))
+ )
+ )
+ if consecutive_errors_len <= 1:
+ return 0
+
+ backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
+ if self.backoff_jitter != 0.0:
+ backoff_value += random.random() * self.backoff_jitter
+ return float(max(0, min(self.backoff_max, backoff_value)))
+
+ def parse_retry_after(self, retry_after: str) -> float:
+ seconds: float
+ # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
+ if re.match(r"^\s*[0-9]+\s*$", retry_after):
+ seconds = int(retry_after)
+ else:
+ retry_date_tuple = email.utils.parsedate_tz(retry_after)
+ if retry_date_tuple is None:
+ raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
+
+ retry_date = email.utils.mktime_tz(retry_date_tuple)
+ seconds = retry_date - time.time()
+
+ seconds = max(seconds, 0)
+
+ return seconds
+
+ def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
+ """Get the value of Retry-After in seconds."""
+
+ retry_after = response.headers.get("Retry-After")
+
+ if retry_after is None:
+ return None
+
+ return self.parse_retry_after(retry_after)
+
+ def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
+ retry_after = self.get_retry_after(response)
+ if retry_after:
+ time.sleep(retry_after)
+ return True
+
+ return False
+
+ def _sleep_backoff(self) -> None:
+ backoff = self.get_backoff_time()
+ if backoff <= 0:
+ return
+ time.sleep(backoff)
+
+ def sleep(self, response: BaseHTTPResponse | None = None) -> None:
+ """Sleep between retry attempts.
+
+ This method will respect a server's ``Retry-After`` response header
+ and sleep the duration of the time requested. If that is not present, it
+ will use an exponential backoff. By default, the backoff factor is 0 and
+ this method will return immediately.
+ """
+
+ if self.respect_retry_after_header and response:
+ slept = self.sleep_for_retry(response)
+ if slept:
+ return
+
+ self._sleep_backoff()
+
+ def _is_connection_error(self, err: Exception) -> bool:
+ """Errors when we're fairly sure that the server did not receive the
+ request, so it should be safe to retry.
+ """
+ if isinstance(err, ProxyError):
+ err = err.original_error
+ return isinstance(err, ConnectTimeoutError)
+
+ def _is_read_error(self, err: Exception) -> bool:
+ """Errors that occur after the request has been started, so we should
+ assume that the server began processing it.
+ """
+ return isinstance(err, (ReadTimeoutError, ProtocolError))
+
+ def _is_method_retryable(self, method: str) -> bool:
+ """Checks if a given HTTP method should be retried upon, depending if
+ it is included in the allowed_methods
+ """
+ if self.allowed_methods and method.upper() not in self.allowed_methods:
+ return False
+ return True
+
+ def is_retry(
+ self, method: str, status_code: int, has_retry_after: bool = False
+ ) -> bool:
+ """Is this method/status code retryable? (Based on allowlists and control
+ variables such as the number of total retries to allow, whether to
+ respect the Retry-After header, whether this header is present, and
+ whether the returned status code is on the list of status codes to
+ be retried upon on the presence of the aforementioned header)
+ """
+ if not self._is_method_retryable(method):
+ return False
+
+ if self.status_forcelist and status_code in self.status_forcelist:
+ return True
+
+ return bool(
+ self.total
+ and self.respect_retry_after_header
+ and has_retry_after
+ and (status_code in self.RETRY_AFTER_STATUS_CODES)
+ )
+
+ def is_exhausted(self) -> bool:
+ """Are we out of retries?"""
+ retry_counts = [
+ x
+ for x in (
+ self.total,
+ self.connect,
+ self.read,
+ self.redirect,
+ self.status,
+ self.other,
+ )
+ if x
+ ]
+ if not retry_counts:
+ return False
+
+ return min(retry_counts) < 0
+
+ def increment(
+ self,
+ method: str | None = None,
+ url: str | None = None,
+ response: BaseHTTPResponse | None = None,
+ error: Exception | None = None,
+ _pool: ConnectionPool | None = None,
+ _stacktrace: TracebackType | None = None,
+ ) -> Self:
+ """Return a new Retry object with incremented retry counters.
+
+ :param response: A response object, or None, if the server did not
+ return a response.
+ :type response: :class:`~urllib3.response.BaseHTTPResponse`
+ :param Exception error: An error encountered during the request, or
+ None if the response was received successfully.
+
+ :return: A new ``Retry`` object.
+ """
+ if self.total is False and error:
+ # Disabled, indicate to re-raise the error.
+ raise reraise(type(error), error, _stacktrace)
+
+ total = self.total
+ if total is not None:
+ total -= 1
+
+ connect = self.connect
+ read = self.read
+ redirect = self.redirect
+ status_count = self.status
+ other = self.other
+ cause = "unknown"
+ status = None
+ redirect_location = None
+
+ if error and self._is_connection_error(error):
+ # Connect retry?
+ if connect is False:
+ raise reraise(type(error), error, _stacktrace)
+ elif connect is not None:
+ connect -= 1
+
+ elif error and self._is_read_error(error):
+ # Read retry?
+ if read is False or method is None or not self._is_method_retryable(method):
+ raise reraise(type(error), error, _stacktrace)
+ elif read is not None:
+ read -= 1
+
+ elif error:
+ # Other retry?
+ if other is not None:
+ other -= 1
+
+ elif response and response.get_redirect_location():
+ # Redirect retry?
+ if redirect is not None:
+ redirect -= 1
+ cause = "too many redirects"
+ response_redirect_location = response.get_redirect_location()
+ if response_redirect_location:
+ redirect_location = response_redirect_location
+ status = response.status
+
+ else:
+ # Incrementing because of a server error like a 500 in
+ # status_forcelist and the given method is in the allowed_methods
+ cause = ResponseError.GENERIC_ERROR
+ if response and response.status:
+ if status_count is not None:
+ status_count -= 1
+ cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
+ status = response.status
+
+ history = self.history + (
+ RequestHistory(method, url, error, status, redirect_location),
+ )
+
+ new_retry = self.new(
+ total=total,
+ connect=connect,
+ read=read,
+ redirect=redirect,
+ status=status_count,
+ other=other,
+ history=history,
+ )
+
+ if new_retry.is_exhausted():
+ reason = error or ResponseError(cause)
+ raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
+
+ log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
+
+ return new_retry
+
+ def __repr__(self) -> str:
+ return (
+ f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
+ f"read={self.read}, redirect={self.redirect}, status={self.status})"
+ )
+
+
+# For backwards compatibility (equivalent to pre-v1.9):
+Retry.DEFAULT = Retry(3)
diff --git a/phivenv/Lib/site-packages/urllib3/util/ssl_.py b/phivenv/Lib/site-packages/urllib3/util/ssl_.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2cc1aa7bb5bb934099bfb44c935b8eccc73f4e0
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/ssl_.py
@@ -0,0 +1,524 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import socket
+import sys
+import typing
+import warnings
+from binascii import unhexlify
+
+from ..exceptions import ProxySchemeUnsupported, SSLError
+from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE
+
+SSLContext = None
+SSLTransport = None
+HAS_NEVER_CHECK_COMMON_NAME = False
+IS_PYOPENSSL = False
+ALPN_PROTOCOLS = ["http/1.1"]
+
+_TYPE_VERSION_INFO = tuple[int, int, int, str, int]
+
+# Maps the length of a digest to a possible hash function producing this digest
+HASHFUNC_MAP = {
+ length: getattr(hashlib, algorithm, None)
+ for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256"))
+}
+
+
+def _is_bpo_43522_fixed(
+ implementation_name: str,
+ version_info: _TYPE_VERSION_INFO,
+ pypy_version_info: _TYPE_VERSION_INFO | None,
+) -> bool:
+ """Return True for CPython 3.9.3+ or 3.10+ and PyPy 7.3.8+ where
+ setting SSLContext.hostname_checks_common_name to False works.
+
+ Outside of CPython and PyPy we don't know which implementations work
+ or not so we conservatively use our hostname matching as we know that works
+ on all implementations.
+
+ https://github.com/urllib3/urllib3/issues/2192#issuecomment-821832963
+ https://foss.heptapod.net/pypy/pypy/-/issues/3539
+ """
+ if implementation_name == "pypy":
+ # https://foss.heptapod.net/pypy/pypy/-/issues/3129
+ return pypy_version_info >= (7, 3, 8) # type: ignore[operator]
+ elif implementation_name == "cpython":
+ major_minor = version_info[:2]
+ micro = version_info[2]
+ return (major_minor == (3, 9) and micro >= 3) or major_minor >= (3, 10)
+ else: # Defensive:
+ return False
+
+
+def _is_has_never_check_common_name_reliable(
+ openssl_version: str,
+ openssl_version_number: int,
+ implementation_name: str,
+ version_info: _TYPE_VERSION_INFO,
+ pypy_version_info: _TYPE_VERSION_INFO | None,
+) -> bool:
+ # As of May 2023, all released versions of LibreSSL fail to reject certificates with
+ # only common names, see https://github.com/urllib3/urllib3/pull/3024
+ is_openssl = openssl_version.startswith("OpenSSL ")
+ # Before fixing OpenSSL issue #14579, the SSL_new() API was not copying hostflags
+ # like X509_CHECK_FLAG_NEVER_CHECK_SUBJECT, which tripped up CPython.
+ # https://github.com/openssl/openssl/issues/14579
+ # This was released in OpenSSL 1.1.1l+ (>=0x101010cf)
+ is_openssl_issue_14579_fixed = openssl_version_number >= 0x101010CF
+
+ return is_openssl and (
+ is_openssl_issue_14579_fixed
+ or _is_bpo_43522_fixed(implementation_name, version_info, pypy_version_info)
+ )
+
+
+if typing.TYPE_CHECKING:
+ from ssl import VerifyMode
+ from typing import TypedDict
+
+ from .ssltransport import SSLTransport as SSLTransportType
+
+ class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False):
+ subjectAltName: tuple[tuple[str, str], ...]
+ subject: tuple[tuple[tuple[str, str], ...], ...]
+ serialNumber: str
+
+
+# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X'
+_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {}
+
+try: # Do we have ssl at all?
+ import ssl
+ from ssl import ( # type: ignore[assignment]
+ CERT_REQUIRED,
+ HAS_NEVER_CHECK_COMMON_NAME,
+ OP_NO_COMPRESSION,
+ OP_NO_TICKET,
+ OPENSSL_VERSION,
+ OPENSSL_VERSION_NUMBER,
+ PROTOCOL_TLS,
+ PROTOCOL_TLS_CLIENT,
+ VERIFY_X509_STRICT,
+ OP_NO_SSLv2,
+ OP_NO_SSLv3,
+ SSLContext,
+ TLSVersion,
+ )
+
+ PROTOCOL_SSLv23 = PROTOCOL_TLS
+
+ # Needed for Python 3.9 which does not define this
+ VERIFY_X509_PARTIAL_CHAIN = getattr(ssl, "VERIFY_X509_PARTIAL_CHAIN", 0x80000)
+
+ # Setting SSLContext.hostname_checks_common_name = False didn't work before CPython
+ # 3.9.3, and 3.10 (but OK on PyPy) or OpenSSL 1.1.1l+
+ if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable(
+ OPENSSL_VERSION,
+ OPENSSL_VERSION_NUMBER,
+ sys.implementation.name,
+ sys.version_info,
+ sys.pypy_version_info if sys.implementation.name == "pypy" else None, # type: ignore[attr-defined]
+ ): # Defensive: for Python < 3.9.3
+ HAS_NEVER_CHECK_COMMON_NAME = False
+
+ # Need to be careful here in case old TLS versions get
+ # removed in future 'ssl' module implementations.
+ for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"):
+ try:
+ _SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr(
+ TLSVersion, attr
+ )
+ except AttributeError: # Defensive:
+ continue
+
+ from .ssltransport import SSLTransport # type: ignore[assignment]
+except ImportError:
+ OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment]
+ OP_NO_TICKET = 0x4000 # type: ignore[assignment]
+ OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment]
+ OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment]
+ PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment]
+ PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment]
+ VERIFY_X509_PARTIAL_CHAIN = 0x80000
+ VERIFY_X509_STRICT = 0x20 # type: ignore[assignment]
+
+
+_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None]
+
+
+def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
+ """
+ Checks if given fingerprint matches the supplied certificate.
+
+ :param cert:
+ Certificate as bytes object.
+ :param fingerprint:
+ Fingerprint as string of hexdigits, can be interspersed by colons.
+ """
+
+ if cert is None:
+ raise SSLError("No certificate for the peer.")
+
+ fingerprint = fingerprint.replace(":", "").lower()
+ digest_length = len(fingerprint)
+ if digest_length not in HASHFUNC_MAP:
+ raise SSLError(f"Fingerprint of invalid length: {fingerprint}")
+ hashfunc = HASHFUNC_MAP.get(digest_length)
+ if hashfunc is None:
+ raise SSLError(
+ f"Hash function implementation unavailable for fingerprint length: {digest_length}"
+ )
+
+ # We need encode() here for py32; works on py2 and p33.
+ fingerprint_bytes = unhexlify(fingerprint.encode())
+
+ cert_digest = hashfunc(cert).digest()
+
+ if not hmac.compare_digest(cert_digest, fingerprint_bytes):
+ raise SSLError(
+ f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"'
+ )
+
+
+def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
+ """
+ Resolves the argument to a numeric constant, which can be passed to
+ the wrap_socket function/method from the ssl module.
+ Defaults to :data:`ssl.CERT_REQUIRED`.
+ If given a string it is assumed to be the name of the constant in the
+ :mod:`ssl` module or its abbreviation.
+ (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
+ If it's neither `None` nor a string we assume it is already the numeric
+ constant which can directly be passed to wrap_socket.
+ """
+ if candidate is None:
+ return CERT_REQUIRED
+
+ if isinstance(candidate, str):
+ res = getattr(ssl, candidate, None)
+ if res is None:
+ res = getattr(ssl, "CERT_" + candidate)
+ return res # type: ignore[no-any-return]
+
+ return candidate # type: ignore[return-value]
+
+
+def resolve_ssl_version(candidate: None | int | str) -> int:
+ """
+ like resolve_cert_reqs
+ """
+ if candidate is None:
+ return PROTOCOL_TLS
+
+ if isinstance(candidate, str):
+ res = getattr(ssl, candidate, None)
+ if res is None:
+ res = getattr(ssl, "PROTOCOL_" + candidate)
+ return typing.cast(int, res)
+
+ return candidate
+
+
+def create_urllib3_context(
+ ssl_version: int | None = None,
+ cert_reqs: int | None = None,
+ options: int | None = None,
+ ciphers: str | None = None,
+ ssl_minimum_version: int | None = None,
+ ssl_maximum_version: int | None = None,
+ verify_flags: int | None = None,
+) -> ssl.SSLContext:
+ """Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3.
+
+ :param ssl_version:
+ The desired protocol version to use. This will default to
+ PROTOCOL_SSLv23 which will negotiate the highest protocol that both
+ the server and your installation of OpenSSL support.
+
+ This parameter is deprecated instead use 'ssl_minimum_version'.
+ :param ssl_minimum_version:
+ The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
+ :param ssl_maximum_version:
+ The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
+ Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the
+ default value.
+ :param cert_reqs:
+ Whether to require the certificate verification. This defaults to
+ ``ssl.CERT_REQUIRED``.
+ :param options:
+ Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
+ ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
+ :param ciphers:
+ Which cipher suites to allow the server to select. Defaults to either system configured
+ ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers.
+ :param verify_flags:
+ The flags for certificate verification operations. These default to
+ ``ssl.VERIFY_X509_PARTIAL_CHAIN`` and ``ssl.VERIFY_X509_STRICT`` for Python 3.13+.
+ :returns:
+ Constructed SSLContext object with specified options
+ :rtype: SSLContext
+ """
+ if SSLContext is None:
+ raise TypeError("Can't create an SSLContext object without an ssl module")
+
+ # This means 'ssl_version' was specified as an exact value.
+ if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT):
+ # Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version'
+ # to avoid conflicts.
+ if ssl_minimum_version is not None or ssl_maximum_version is not None:
+ raise ValueError(
+ "Can't specify both 'ssl_version' and either "
+ "'ssl_minimum_version' or 'ssl_maximum_version'"
+ )
+
+ # 'ssl_version' is deprecated and will be removed in the future.
+ else:
+ # Use 'ssl_minimum_version' and 'ssl_maximum_version' instead.
+ ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get(
+ ssl_version, TLSVersion.MINIMUM_SUPPORTED
+ )
+ ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get(
+ ssl_version, TLSVersion.MAXIMUM_SUPPORTED
+ )
+
+ # This warning message is pushing users to use 'ssl_minimum_version'
+ # instead of both min/max. Best practice is to only set the minimum version and
+ # keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED'
+ warnings.warn(
+ "'ssl_version' option is deprecated and will be "
+ "removed in urllib3 v2.6.0. Instead use 'ssl_minimum_version'",
+ category=DeprecationWarning,
+ stacklevel=2,
+ )
+
+ # PROTOCOL_TLS is deprecated in Python 3.10 so we always use PROTOCOL_TLS_CLIENT
+ context = SSLContext(PROTOCOL_TLS_CLIENT)
+
+ if ssl_minimum_version is not None:
+ context.minimum_version = ssl_minimum_version
+ else: # Python <3.10 defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here
+ context.minimum_version = TLSVersion.TLSv1_2
+
+ if ssl_maximum_version is not None:
+ context.maximum_version = ssl_maximum_version
+
+ # Unless we're given ciphers defer to either system ciphers in
+ # the case of OpenSSL 1.1.1+ or use our own secure default ciphers.
+ if ciphers:
+ context.set_ciphers(ciphers)
+
+ # Setting the default here, as we may have no ssl module on import
+ cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
+
+ if options is None:
+ options = 0
+ # SSLv2 is easily broken and is considered harmful and dangerous
+ options |= OP_NO_SSLv2
+ # SSLv3 has several problems and is now dangerous
+ options |= OP_NO_SSLv3
+ # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
+ # (issue #309)
+ options |= OP_NO_COMPRESSION
+ # TLSv1.2 only. Unless set explicitly, do not request tickets.
+ # This may save some bandwidth on wire, and although the ticket is encrypted,
+ # there is a risk associated with it being on wire,
+ # if the server is not rotating its ticketing keys properly.
+ options |= OP_NO_TICKET
+
+ context.options |= options
+
+ if verify_flags is None:
+ verify_flags = 0
+ # In Python 3.13+ ssl.create_default_context() sets VERIFY_X509_PARTIAL_CHAIN
+ # and VERIFY_X509_STRICT so we do the same
+ if sys.version_info >= (3, 13):
+ verify_flags |= VERIFY_X509_PARTIAL_CHAIN
+ verify_flags |= VERIFY_X509_STRICT
+
+ context.verify_flags |= verify_flags
+
+ # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
+ # necessary for conditional client cert authentication with TLS 1.3.
+ # The attribute is None for OpenSSL <= 1.1.0 or does not exist when using
+ # an SSLContext created by pyOpenSSL.
+ if getattr(context, "post_handshake_auth", None) is not None:
+ context.post_handshake_auth = True
+
+ # The order of the below lines setting verify_mode and check_hostname
+ # matter due to safe-guards SSLContext has to prevent an SSLContext with
+ # check_hostname=True, verify_mode=NONE/OPTIONAL.
+ # We always set 'check_hostname=False' for pyOpenSSL so we rely on our own
+ # 'ssl.match_hostname()' implementation.
+ if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL:
+ context.verify_mode = cert_reqs
+ context.check_hostname = True
+ else:
+ context.check_hostname = False
+ context.verify_mode = cert_reqs
+
+ try:
+ context.hostname_checks_common_name = False
+ except AttributeError: # Defensive: for CPython < 3.9.3; for PyPy < 7.3.8
+ pass
+
+ sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
+ if sslkeylogfile:
+ context.keylog_filename = sslkeylogfile
+
+ return context
+
+
+@typing.overload
+def ssl_wrap_socket(
+ sock: socket.socket,
+ keyfile: str | None = ...,
+ certfile: str | None = ...,
+ cert_reqs: int | None = ...,
+ ca_certs: str | None = ...,
+ server_hostname: str | None = ...,
+ ssl_version: int | None = ...,
+ ciphers: str | None = ...,
+ ssl_context: ssl.SSLContext | None = ...,
+ ca_cert_dir: str | None = ...,
+ key_password: str | None = ...,
+ ca_cert_data: None | str | bytes = ...,
+ tls_in_tls: typing.Literal[False] = ...,
+) -> ssl.SSLSocket: ...
+
+
+@typing.overload
+def ssl_wrap_socket(
+ sock: socket.socket,
+ keyfile: str | None = ...,
+ certfile: str | None = ...,
+ cert_reqs: int | None = ...,
+ ca_certs: str | None = ...,
+ server_hostname: str | None = ...,
+ ssl_version: int | None = ...,
+ ciphers: str | None = ...,
+ ssl_context: ssl.SSLContext | None = ...,
+ ca_cert_dir: str | None = ...,
+ key_password: str | None = ...,
+ ca_cert_data: None | str | bytes = ...,
+ tls_in_tls: bool = ...,
+) -> ssl.SSLSocket | SSLTransportType: ...
+
+
+def ssl_wrap_socket(
+ sock: socket.socket,
+ keyfile: str | None = None,
+ certfile: str | None = None,
+ cert_reqs: int | None = None,
+ ca_certs: str | None = None,
+ server_hostname: str | None = None,
+ ssl_version: int | None = None,
+ ciphers: str | None = None,
+ ssl_context: ssl.SSLContext | None = None,
+ ca_cert_dir: str | None = None,
+ key_password: str | None = None,
+ ca_cert_data: None | str | bytes = None,
+ tls_in_tls: bool = False,
+) -> ssl.SSLSocket | SSLTransportType:
+ """
+ All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and
+ ca_cert_dir have the same meaning as they do when using
+ :func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`,
+ :meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`.
+
+ :param server_hostname:
+ When SNI is supported, the expected hostname of the certificate
+ :param ssl_context:
+ A pre-made :class:`SSLContext` object. If none is provided, one will
+ be created using :func:`create_urllib3_context`.
+ :param ciphers:
+ A string of ciphers we wish the client to support.
+ :param ca_cert_dir:
+ A directory containing CA certificates in multiple separate files, as
+ supported by OpenSSL's -CApath flag or the capath argument to
+ SSLContext.load_verify_locations().
+ :param key_password:
+ Optional password if the keyfile is encrypted.
+ :param ca_cert_data:
+ Optional string containing CA certificates in PEM format suitable for
+ passing as the cadata parameter to SSLContext.load_verify_locations()
+ :param tls_in_tls:
+ Use SSLTransport to wrap the existing socket.
+ """
+ context = ssl_context
+ if context is None:
+ # Note: This branch of code and all the variables in it are only used in tests.
+ # We should consider deprecating and removing this code.
+ context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
+
+ if ca_certs or ca_cert_dir or ca_cert_data:
+ try:
+ context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
+ except OSError as e:
+ raise SSLError(e) from e
+
+ elif ssl_context is None and hasattr(context, "load_default_certs"):
+ # try to load OS default certs; works well on Windows.
+ context.load_default_certs()
+
+ # Attempt to detect if we get the goofy behavior of the
+ # keyfile being encrypted and OpenSSL asking for the
+ # passphrase via the terminal and instead error out.
+ if keyfile and key_password is None and _is_key_file_encrypted(keyfile):
+ raise SSLError("Client private key is encrypted, password is required")
+
+ if certfile:
+ if key_password is None:
+ context.load_cert_chain(certfile, keyfile)
+ else:
+ context.load_cert_chain(certfile, keyfile, key_password)
+
+ context.set_alpn_protocols(ALPN_PROTOCOLS)
+
+ ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
+ return ssl_sock
+
+
+def is_ipaddress(hostname: str | bytes) -> bool:
+ """Detects whether the hostname given is an IPv4 or IPv6 address.
+ Also detects IPv6 addresses with Zone IDs.
+
+ :param str hostname: Hostname to examine.
+ :return: True if the hostname is an IP address, False otherwise.
+ """
+ if isinstance(hostname, bytes):
+ # IDN A-label bytes are ASCII compatible.
+ hostname = hostname.decode("ascii")
+ return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname))
+
+
+def _is_key_file_encrypted(key_file: str) -> bool:
+ """Detects if a key file is encrypted or not."""
+ with open(key_file) as f:
+ for line in f:
+ # Look for Proc-Type: 4,ENCRYPTED
+ if "ENCRYPTED" in line:
+ return True
+
+ return False
+
+
+def _ssl_wrap_socket_impl(
+ sock: socket.socket,
+ ssl_context: ssl.SSLContext,
+ tls_in_tls: bool,
+ server_hostname: str | None = None,
+) -> ssl.SSLSocket | SSLTransportType:
+ if tls_in_tls:
+ if not SSLTransport:
+ # Import error, ssl is not available.
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires support for the 'ssl' module"
+ )
+
+ SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
+ return SSLTransport(sock, ssl_context, server_hostname)
+
+ return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
diff --git a/phivenv/Lib/site-packages/urllib3/util/ssl_match_hostname.py b/phivenv/Lib/site-packages/urllib3/util/ssl_match_hostname.py
new file mode 100644
index 0000000000000000000000000000000000000000..25d91000419ea4a860f511ebe669fe171b79254c
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/ssl_match_hostname.py
@@ -0,0 +1,159 @@
+"""The match_hostname() function from Python 3.5, essential when using SSL."""
+
+# Note: This file is under the PSF license as the code comes from the python
+# stdlib. http://docs.python.org/3/license.html
+# It is modified to remove commonName support.
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import typing
+from ipaddress import IPv4Address, IPv6Address
+
+if typing.TYPE_CHECKING:
+ from .ssl_ import _TYPE_PEER_CERT_RET_DICT
+
+__version__ = "3.5.0.1"
+
+
+class CertificateError(ValueError):
+ pass
+
+
+def _dnsname_match(
+ dn: typing.Any, hostname: str, max_wildcards: int = 1
+) -> typing.Match[str] | None | bool:
+ """Matching according to RFC 6125, section 6.4.3
+
+ http://tools.ietf.org/html/rfc6125#section-6.4.3
+ """
+ pats = []
+ if not dn:
+ return False
+
+ # Ported from python3-syntax:
+ # leftmost, *remainder = dn.split(r'.')
+ parts = dn.split(r".")
+ leftmost = parts[0]
+ remainder = parts[1:]
+
+ wildcards = leftmost.count("*")
+ if wildcards > max_wildcards:
+ # Issue #17980: avoid denials of service by refusing more
+ # than one wildcard per fragment. A survey of established
+ # policy among SSL implementations showed it to be a
+ # reasonable choice.
+ raise CertificateError(
+ "too many wildcards in certificate DNS name: " + repr(dn)
+ )
+
+ # speed up common case w/o wildcards
+ if not wildcards:
+ return bool(dn.lower() == hostname.lower())
+
+ # RFC 6125, section 6.4.3, subitem 1.
+ # The client SHOULD NOT attempt to match a presented identifier in which
+ # the wildcard character comprises a label other than the left-most label.
+ if leftmost == "*":
+ # When '*' is a fragment by itself, it matches a non-empty dotless
+ # fragment.
+ pats.append("[^.]+")
+ elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
+ # RFC 6125, section 6.4.3, subitem 3.
+ # The client SHOULD NOT attempt to match a presented identifier
+ # where the wildcard character is embedded within an A-label or
+ # U-label of an internationalized domain name.
+ pats.append(re.escape(leftmost))
+ else:
+ # Otherwise, '*' matches any dotless string, e.g. www*
+ pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
+
+ # add the remaining fragments, ignore any wildcards
+ for frag in remainder:
+ pats.append(re.escape(frag))
+
+ pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
+ return pat.match(hostname)
+
+
+def _ipaddress_match(ipname: str, host_ip: IPv4Address | IPv6Address) -> bool:
+ """Exact matching of IP addresses.
+
+ RFC 9110 section 4.3.5: "A reference identity of IP-ID contains the decoded
+ bytes of the IP address. An IP version 4 address is 4 octets, and an IP
+ version 6 address is 16 octets. [...] A reference identity of type IP-ID
+ matches if the address is identical to an iPAddress value of the
+ subjectAltName extension of the certificate."
+ """
+ # OpenSSL may add a trailing newline to a subjectAltName's IP address
+ # Divergence from upstream: ipaddress can't handle byte str
+ ip = ipaddress.ip_address(ipname.rstrip())
+ return bool(ip.packed == host_ip.packed)
+
+
+def match_hostname(
+ cert: _TYPE_PEER_CERT_RET_DICT | None,
+ hostname: str,
+ hostname_checks_common_name: bool = False,
+) -> None:
+ """Verify that *cert* (in decoded format as returned by
+ SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125
+ rules are followed, but IP addresses are not accepted for *hostname*.
+
+ CertificateError is raised on failure. On success, the function
+ returns nothing.
+ """
+ if not cert:
+ raise ValueError(
+ "empty or no certificate, match_hostname needs a "
+ "SSL socket or SSL context with either "
+ "CERT_OPTIONAL or CERT_REQUIRED"
+ )
+ try:
+ # Divergence from upstream: ipaddress can't handle byte str
+ #
+ # The ipaddress module shipped with Python < 3.9 does not support
+ # scoped IPv6 addresses so we unconditionally strip the Zone IDs for
+ # now. Once we drop support for Python 3.9 we can remove this branch.
+ if "%" in hostname:
+ host_ip = ipaddress.ip_address(hostname[: hostname.rfind("%")])
+ else:
+ host_ip = ipaddress.ip_address(hostname)
+
+ except ValueError:
+ # Not an IP address (common case)
+ host_ip = None
+ dnsnames = []
+ san: tuple[tuple[str, str], ...] = cert.get("subjectAltName", ())
+ key: str
+ value: str
+ for key, value in san:
+ if key == "DNS":
+ if host_ip is None and _dnsname_match(value, hostname):
+ return
+ dnsnames.append(value)
+ elif key == "IP Address":
+ if host_ip is not None and _ipaddress_match(value, host_ip):
+ return
+ dnsnames.append(value)
+
+ # We only check 'commonName' if it's enabled and we're not verifying
+ # an IP address. IP addresses aren't valid within 'commonName'.
+ if hostname_checks_common_name and host_ip is None and not dnsnames:
+ for sub in cert.get("subject", ()):
+ for key, value in sub:
+ if key == "commonName":
+ if _dnsname_match(value, hostname):
+ return
+ dnsnames.append(value) # Defensive: for Python < 3.9.3
+
+ if len(dnsnames) > 1:
+ raise CertificateError(
+ "hostname %r "
+ "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
+ )
+ elif len(dnsnames) == 1:
+ raise CertificateError(f"hostname {hostname!r} doesn't match {dnsnames[0]!r}")
+ else:
+ raise CertificateError("no appropriate subjectAltName fields were found")
diff --git a/phivenv/Lib/site-packages/urllib3/util/ssltransport.py b/phivenv/Lib/site-packages/urllib3/util/ssltransport.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d59bc3bce2489c3a0aa5bcb83b737dcf33c033b
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/ssltransport.py
@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import io
+import socket
+import ssl
+import typing
+
+from ..exceptions import ProxySchemeUnsupported
+
+if typing.TYPE_CHECKING:
+ from typing_extensions import Self
+
+ from .ssl_ import _TYPE_PEER_CERT_RET, _TYPE_PEER_CERT_RET_DICT
+
+
+_WriteBuffer = typing.Union[bytearray, memoryview]
+_ReturnValue = typing.TypeVar("_ReturnValue")
+
+SSL_BLOCKSIZE = 16384
+
+
+class SSLTransport:
+ """
+ The SSLTransport wraps an existing socket and establishes an SSL connection.
+
+ Contrary to Python's implementation of SSLSocket, it allows you to chain
+ multiple TLS connections together. It's particularly useful if you need to
+ implement TLS within TLS.
+
+ The class supports most of the socket API operations.
+ """
+
+ @staticmethod
+ def _validate_ssl_context_for_tls_in_tls(ssl_context: ssl.SSLContext) -> None:
+ """
+ Raises a ProxySchemeUnsupported if the provided ssl_context can't be used
+ for TLS in TLS.
+
+ The only requirement is that the ssl_context provides the 'wrap_bio'
+ methods.
+ """
+
+ if not hasattr(ssl_context, "wrap_bio"):
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires SSLContext.wrap_bio() which isn't "
+ "available on non-native SSLContext"
+ )
+
+ def __init__(
+ self,
+ socket: socket.socket,
+ ssl_context: ssl.SSLContext,
+ server_hostname: str | None = None,
+ suppress_ragged_eofs: bool = True,
+ ) -> None:
+ """
+ Create an SSLTransport around socket using the provided ssl_context.
+ """
+ self.incoming = ssl.MemoryBIO()
+ self.outgoing = ssl.MemoryBIO()
+
+ self.suppress_ragged_eofs = suppress_ragged_eofs
+ self.socket = socket
+
+ self.sslobj = ssl_context.wrap_bio(
+ self.incoming, self.outgoing, server_hostname=server_hostname
+ )
+
+ # Perform initial handshake.
+ self._ssl_io_loop(self.sslobj.do_handshake)
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(self, *_: typing.Any) -> None:
+ self.close()
+
+ def fileno(self) -> int:
+ return self.socket.fileno()
+
+ def read(self, len: int = 1024, buffer: typing.Any | None = None) -> int | bytes:
+ return self._wrap_ssl_read(len, buffer)
+
+ def recv(self, buflen: int = 1024, flags: int = 0) -> int | bytes:
+ if flags != 0:
+ raise ValueError("non-zero flags not allowed in calls to recv")
+ return self._wrap_ssl_read(buflen)
+
+ def recv_into(
+ self,
+ buffer: _WriteBuffer,
+ nbytes: int | None = None,
+ flags: int = 0,
+ ) -> None | int | bytes:
+ if flags != 0:
+ raise ValueError("non-zero flags not allowed in calls to recv_into")
+ if nbytes is None:
+ nbytes = len(buffer)
+ return self.read(nbytes, buffer)
+
+ def sendall(self, data: bytes, flags: int = 0) -> None:
+ if flags != 0:
+ raise ValueError("non-zero flags not allowed in calls to sendall")
+ count = 0
+ with memoryview(data) as view, view.cast("B") as byte_view:
+ amount = len(byte_view)
+ while count < amount:
+ v = self.send(byte_view[count:])
+ count += v
+
+ def send(self, data: bytes, flags: int = 0) -> int:
+ if flags != 0:
+ raise ValueError("non-zero flags not allowed in calls to send")
+ return self._ssl_io_loop(self.sslobj.write, data)
+
+ def makefile(
+ self,
+ mode: str,
+ buffering: int | None = None,
+ *,
+ encoding: str | None = None,
+ errors: str | None = None,
+ newline: str | None = None,
+ ) -> typing.BinaryIO | typing.TextIO | socket.SocketIO:
+ """
+ Python's httpclient uses makefile and buffered io when reading HTTP
+ messages and we need to support it.
+
+ This is unfortunately a copy and paste of socket.py makefile with small
+ changes to point to the socket directly.
+ """
+ if not set(mode) <= {"r", "w", "b"}:
+ raise ValueError(f"invalid mode {mode!r} (only r, w, b allowed)")
+
+ writing = "w" in mode
+ reading = "r" in mode or not writing
+ assert reading or writing
+ binary = "b" in mode
+ rawmode = ""
+ if reading:
+ rawmode += "r"
+ if writing:
+ rawmode += "w"
+ raw = socket.SocketIO(self, rawmode) # type: ignore[arg-type]
+ self.socket._io_refs += 1 # type: ignore[attr-defined]
+ if buffering is None:
+ buffering = -1
+ if buffering < 0:
+ buffering = io.DEFAULT_BUFFER_SIZE
+ if buffering == 0:
+ if not binary:
+ raise ValueError("unbuffered streams must be binary")
+ return raw
+ buffer: typing.BinaryIO
+ if reading and writing:
+ buffer = io.BufferedRWPair(raw, raw, buffering) # type: ignore[assignment]
+ elif reading:
+ buffer = io.BufferedReader(raw, buffering)
+ else:
+ assert writing
+ buffer = io.BufferedWriter(raw, buffering)
+ if binary:
+ return buffer
+ text = io.TextIOWrapper(buffer, encoding, errors, newline)
+ text.mode = mode # type: ignore[misc]
+ return text
+
+ def unwrap(self) -> None:
+ self._ssl_io_loop(self.sslobj.unwrap)
+
+ def close(self) -> None:
+ self.socket.close()
+
+ @typing.overload
+ def getpeercert(
+ self, binary_form: typing.Literal[False] = ...
+ ) -> _TYPE_PEER_CERT_RET_DICT | None: ...
+
+ @typing.overload
+ def getpeercert(self, binary_form: typing.Literal[True]) -> bytes | None: ...
+
+ def getpeercert(self, binary_form: bool = False) -> _TYPE_PEER_CERT_RET:
+ return self.sslobj.getpeercert(binary_form) # type: ignore[return-value]
+
+ def version(self) -> str | None:
+ return self.sslobj.version()
+
+ def cipher(self) -> tuple[str, str, int] | None:
+ return self.sslobj.cipher()
+
+ def selected_alpn_protocol(self) -> str | None:
+ return self.sslobj.selected_alpn_protocol()
+
+ def shared_ciphers(self) -> list[tuple[str, str, int]] | None:
+ return self.sslobj.shared_ciphers()
+
+ def compression(self) -> str | None:
+ return self.sslobj.compression()
+
+ def settimeout(self, value: float | None) -> None:
+ self.socket.settimeout(value)
+
+ def gettimeout(self) -> float | None:
+ return self.socket.gettimeout()
+
+ def _decref_socketios(self) -> None:
+ self.socket._decref_socketios() # type: ignore[attr-defined]
+
+ def _wrap_ssl_read(self, len: int, buffer: bytearray | None = None) -> int | bytes:
+ try:
+ return self._ssl_io_loop(self.sslobj.read, len, buffer)
+ except ssl.SSLError as e:
+ if e.errno == ssl.SSL_ERROR_EOF and self.suppress_ragged_eofs:
+ return 0 # eof, return 0.
+ else:
+ raise
+
+ # func is sslobj.do_handshake or sslobj.unwrap
+ @typing.overload
+ def _ssl_io_loop(self, func: typing.Callable[[], None]) -> None: ...
+
+ # func is sslobj.write, arg1 is data
+ @typing.overload
+ def _ssl_io_loop(self, func: typing.Callable[[bytes], int], arg1: bytes) -> int: ...
+
+ # func is sslobj.read, arg1 is len, arg2 is buffer
+ @typing.overload
+ def _ssl_io_loop(
+ self,
+ func: typing.Callable[[int, bytearray | None], bytes],
+ arg1: int,
+ arg2: bytearray | None,
+ ) -> bytes: ...
+
+ def _ssl_io_loop(
+ self,
+ func: typing.Callable[..., _ReturnValue],
+ arg1: None | bytes | int = None,
+ arg2: bytearray | None = None,
+ ) -> _ReturnValue:
+ """Performs an I/O loop between incoming/outgoing and the socket."""
+ should_loop = True
+ ret = None
+
+ while should_loop:
+ errno = None
+ try:
+ if arg1 is None and arg2 is None:
+ ret = func()
+ elif arg2 is None:
+ ret = func(arg1)
+ else:
+ ret = func(arg1, arg2)
+ except ssl.SSLError as e:
+ if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE):
+ # WANT_READ, and WANT_WRITE are expected, others are not.
+ raise e
+ errno = e.errno
+
+ buf = self.outgoing.read()
+ self.socket.sendall(buf)
+
+ if errno is None:
+ should_loop = False
+ elif errno == ssl.SSL_ERROR_WANT_READ:
+ buf = self.socket.recv(SSL_BLOCKSIZE)
+ if buf:
+ self.incoming.write(buf)
+ else:
+ self.incoming.write_eof()
+ return typing.cast(_ReturnValue, ret)
diff --git a/phivenv/Lib/site-packages/urllib3/util/timeout.py b/phivenv/Lib/site-packages/urllib3/util/timeout.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bb1be11d9cb06900dd82ecebd06aa6a7c5de916
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/timeout.py
@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import time
+import typing
+from enum import Enum
+from socket import getdefaulttimeout
+
+from ..exceptions import TimeoutStateError
+
+if typing.TYPE_CHECKING:
+ from typing import Final
+
+
+class _TYPE_DEFAULT(Enum):
+ # This value should never be passed to socket.settimeout() so for safety we use a -1.
+ # socket.settimout() raises a ValueError for negative values.
+ token = -1
+
+
+_DEFAULT_TIMEOUT: Final[_TYPE_DEFAULT] = _TYPE_DEFAULT.token
+
+_TYPE_TIMEOUT = typing.Optional[typing.Union[float, _TYPE_DEFAULT]]
+
+
+class Timeout:
+ """Timeout configuration.
+
+ Timeouts can be defined as a default for a pool:
+
+ .. code-block:: python
+
+ import urllib3
+
+ timeout = urllib3.util.Timeout(connect=2.0, read=7.0)
+
+ http = urllib3.PoolManager(timeout=timeout)
+
+ resp = http.request("GET", "https://example.com/")
+
+ print(resp.status)
+
+ Or per-request (which overrides the default for the pool):
+
+ .. code-block:: python
+
+ response = http.request("GET", "https://example.com/", timeout=Timeout(10))
+
+ Timeouts can be disabled by setting all the parameters to ``None``:
+
+ .. code-block:: python
+
+ no_timeout = Timeout(connect=None, read=None)
+ response = http.request("GET", "https://example.com/", timeout=no_timeout)
+
+
+ :param total:
+ This combines the connect and read timeouts into one; the read timeout
+ will be set to the time leftover from the connect attempt. In the
+ event that both a connect timeout and a total are specified, or a read
+ timeout and a total are specified, the shorter timeout will be applied.
+
+ Defaults to None.
+
+ :type total: int, float, or None
+
+ :param connect:
+ The maximum amount of time (in seconds) to wait for a connection
+ attempt to a server to succeed. Omitting the parameter will default the
+ connect timeout to the system default, probably `the global default
+ timeout in socket.py
+ `_.
+ None will set an infinite timeout for connection attempts.
+
+ :type connect: int, float, or None
+
+ :param read:
+ The maximum amount of time (in seconds) to wait between consecutive
+ read operations for a response from the server. Omitting the parameter
+ will default the read timeout to the system default, probably `the
+ global default timeout in socket.py
+ `_.
+ None will set an infinite timeout.
+
+ :type read: int, float, or None
+
+ .. note::
+
+ Many factors can affect the total amount of time for urllib3 to return
+ an HTTP response.
+
+ For example, Python's DNS resolver does not obey the timeout specified
+ on the socket. Other factors that can affect total request time include
+ high CPU load, high swap, the program running at a low priority level,
+ or other behaviors.
+
+ In addition, the read and total timeouts only measure the time between
+ read operations on the socket connecting the client and the server,
+ not the total amount of time for the request to return a complete
+ response. For most requests, the timeout is raised because the server
+ has not sent the first byte in the specified time. This is not always
+ the case; if a server streams one byte every fifteen seconds, a timeout
+ of 20 seconds will not trigger, even though the request will take
+ several minutes to complete.
+ """
+
+ #: A sentinel object representing the default timeout value
+ DEFAULT_TIMEOUT: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT
+
+ def __init__(
+ self,
+ total: _TYPE_TIMEOUT = None,
+ connect: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ read: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
+ ) -> None:
+ self._connect = self._validate_timeout(connect, "connect")
+ self._read = self._validate_timeout(read, "read")
+ self.total = self._validate_timeout(total, "total")
+ self._start_connect: float | None = None
+
+ def __repr__(self) -> str:
+ return f"{type(self).__name__}(connect={self._connect!r}, read={self._read!r}, total={self.total!r})"
+
+ # __str__ provided for backwards compatibility
+ __str__ = __repr__
+
+ @staticmethod
+ def resolve_default_timeout(timeout: _TYPE_TIMEOUT) -> float | None:
+ return getdefaulttimeout() if timeout is _DEFAULT_TIMEOUT else timeout
+
+ @classmethod
+ def _validate_timeout(cls, value: _TYPE_TIMEOUT, name: str) -> _TYPE_TIMEOUT:
+ """Check that a timeout attribute is valid.
+
+ :param value: The timeout value to validate
+ :param name: The name of the timeout attribute to validate. This is
+ used to specify in error messages.
+ :return: The validated and casted version of the given value.
+ :raises ValueError: If it is a numeric value less than or equal to
+ zero, or the type is not an integer, float, or None.
+ """
+ if value is None or value is _DEFAULT_TIMEOUT:
+ return value
+
+ if isinstance(value, bool):
+ raise ValueError(
+ "Timeout cannot be a boolean value. It must "
+ "be an int, float or None."
+ )
+ try:
+ float(value)
+ except (TypeError, ValueError):
+ raise ValueError(
+ "Timeout value %s was %s, but it must be an "
+ "int, float or None." % (name, value)
+ ) from None
+
+ try:
+ if value <= 0:
+ raise ValueError(
+ "Attempted to set %s timeout to %s, but the "
+ "timeout cannot be set to a value less "
+ "than or equal to 0." % (name, value)
+ )
+ except TypeError:
+ raise ValueError(
+ "Timeout value %s was %s, but it must be an "
+ "int, float or None." % (name, value)
+ ) from None
+
+ return value
+
+ @classmethod
+ def from_float(cls, timeout: _TYPE_TIMEOUT) -> Timeout:
+ """Create a new Timeout from a legacy timeout value.
+
+ The timeout value used by httplib.py sets the same timeout on the
+ connect(), and recv() socket requests. This creates a :class:`Timeout`
+ object that sets the individual timeouts to the ``timeout`` value
+ passed to this function.
+
+ :param timeout: The legacy timeout value.
+ :type timeout: integer, float, :attr:`urllib3.util.Timeout.DEFAULT_TIMEOUT`, or None
+ :return: Timeout object
+ :rtype: :class:`Timeout`
+ """
+ return Timeout(read=timeout, connect=timeout)
+
+ def clone(self) -> Timeout:
+ """Create a copy of the timeout object
+
+ Timeout properties are stored per-pool but each request needs a fresh
+ Timeout object to ensure each one has its own start/stop configured.
+
+ :return: a copy of the timeout object
+ :rtype: :class:`Timeout`
+ """
+ # We can't use copy.deepcopy because that will also create a new object
+ # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
+ # detect the user default.
+ return Timeout(connect=self._connect, read=self._read, total=self.total)
+
+ def start_connect(self) -> float:
+ """Start the timeout clock, used during a connect() attempt
+
+ :raises urllib3.exceptions.TimeoutStateError: if you attempt
+ to start a timer that has been started already.
+ """
+ if self._start_connect is not None:
+ raise TimeoutStateError("Timeout timer has already been started.")
+ self._start_connect = time.monotonic()
+ return self._start_connect
+
+ def get_connect_duration(self) -> float:
+ """Gets the time elapsed since the call to :meth:`start_connect`.
+
+ :return: Elapsed time in seconds.
+ :rtype: float
+ :raises urllib3.exceptions.TimeoutStateError: if you attempt
+ to get duration for a timer that hasn't been started.
+ """
+ if self._start_connect is None:
+ raise TimeoutStateError(
+ "Can't get connect duration for timer that has not started."
+ )
+ return time.monotonic() - self._start_connect
+
+ @property
+ def connect_timeout(self) -> _TYPE_TIMEOUT:
+ """Get the value to use when setting a connection timeout.
+
+ This will be a positive float or integer, the value None
+ (never timeout), or the default system timeout.
+
+ :return: Connect timeout.
+ :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
+ """
+ if self.total is None:
+ return self._connect
+
+ if self._connect is None or self._connect is _DEFAULT_TIMEOUT:
+ return self.total
+
+ return min(self._connect, self.total) # type: ignore[type-var]
+
+ @property
+ def read_timeout(self) -> float | None:
+ """Get the value for the read timeout.
+
+ This assumes some time has elapsed in the connection timeout and
+ computes the read timeout appropriately.
+
+ If self.total is set, the read timeout is dependent on the amount of
+ time taken by the connect timeout. If the connection time has not been
+ established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
+ raised.
+
+ :return: Value to use for the read timeout.
+ :rtype: int, float or None
+ :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
+ has not yet been called on this object.
+ """
+ if (
+ self.total is not None
+ and self.total is not _DEFAULT_TIMEOUT
+ and self._read is not None
+ and self._read is not _DEFAULT_TIMEOUT
+ ):
+ # In case the connect timeout has not yet been established.
+ if self._start_connect is None:
+ return self._read
+ return max(0, min(self.total - self.get_connect_duration(), self._read))
+ elif self.total is not None and self.total is not _DEFAULT_TIMEOUT:
+ return max(0, self.total - self.get_connect_duration())
+ else:
+ return self.resolve_default_timeout(self._read)
diff --git a/phivenv/Lib/site-packages/urllib3/util/url.py b/phivenv/Lib/site-packages/urllib3/util/url.py
new file mode 100644
index 0000000000000000000000000000000000000000..db057f17be610174f30928748b5004dcbf6c501c
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/url.py
@@ -0,0 +1,469 @@
+from __future__ import annotations
+
+import re
+import typing
+
+from ..exceptions import LocationParseError
+from .util import to_str
+
+# We only want to normalize urls with an HTTP(S) scheme.
+# urllib3 infers URLs without a scheme (None) to be http.
+_NORMALIZABLE_SCHEMES = ("http", "https", None)
+
+# Almost all of these patterns were derived from the
+# 'rfc3986' module: https://github.com/python-hyper/rfc3986
+_PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}")
+_SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)")
+_URI_RE = re.compile(
+ r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?"
+ r"(?://([^\\/?#]*))?"
+ r"([^?#]*)"
+ r"(?:\?([^#]*))?"
+ r"(?:#(.*))?$",
+ re.UNICODE | re.DOTALL,
+)
+
+_IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"
+_HEX_PAT = "[0-9A-Fa-f]{1,4}"
+_LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=_HEX_PAT, ipv4=_IPV4_PAT)
+_subs = {"hex": _HEX_PAT, "ls32": _LS32_PAT}
+_variations = [
+ # 6( h16 ":" ) ls32
+ "(?:%(hex)s:){6}%(ls32)s",
+ # "::" 5( h16 ":" ) ls32
+ "::(?:%(hex)s:){5}%(ls32)s",
+ # [ h16 ] "::" 4( h16 ":" ) ls32
+ "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",
+ # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
+ "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",
+ # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
+ "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",
+ # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
+ "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",
+ # [ *4( h16 ":" ) h16 ] "::" ls32
+ "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",
+ # [ *5( h16 ":" ) h16 ] "::" h16
+ "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",
+ # [ *6( h16 ":" ) h16 ] "::"
+ "(?:(?:%(hex)s:){0,6}%(hex)s)?::",
+]
+
+_UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._\-~"
+_IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"
+_ZONE_ID_PAT = "(?:%25|%)(?:[" + _UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"
+_IPV6_ADDRZ_PAT = r"\[" + _IPV6_PAT + r"(?:" + _ZONE_ID_PAT + r")?\]"
+_REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*"
+_TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$")
+
+_IPV4_RE = re.compile("^" + _IPV4_PAT + "$")
+_IPV6_RE = re.compile("^" + _IPV6_PAT + "$")
+_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT + "$")
+_BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + _IPV6_ADDRZ_PAT[2:-2] + "$")
+_ZONE_ID_RE = re.compile("(" + _ZONE_ID_PAT + r")\]$")
+
+_HOST_PORT_PAT = ("^(%s|%s|%s)(?::0*?(|0|[1-9][0-9]{0,4}))?$") % (
+ _REG_NAME_PAT,
+ _IPV4_PAT,
+ _IPV6_ADDRZ_PAT,
+)
+_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL)
+
+_UNRESERVED_CHARS = set(
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~"
+)
+_SUB_DELIM_CHARS = set("!$&'()*+,;=")
+_USERINFO_CHARS = _UNRESERVED_CHARS | _SUB_DELIM_CHARS | {":"}
+_PATH_CHARS = _USERINFO_CHARS | {"@", "/"}
+_QUERY_CHARS = _FRAGMENT_CHARS = _PATH_CHARS | {"?"}
+
+
+class Url(
+ typing.NamedTuple(
+ "Url",
+ [
+ ("scheme", typing.Optional[str]),
+ ("auth", typing.Optional[str]),
+ ("host", typing.Optional[str]),
+ ("port", typing.Optional[int]),
+ ("path", typing.Optional[str]),
+ ("query", typing.Optional[str]),
+ ("fragment", typing.Optional[str]),
+ ],
+ )
+):
+ """
+ Data structure for representing an HTTP URL. Used as a return value for
+ :func:`parse_url`. Both the scheme and host are normalized as they are
+ both case-insensitive according to RFC 3986.
+ """
+
+ def __new__( # type: ignore[no-untyped-def]
+ cls,
+ scheme: str | None = None,
+ auth: str | None = None,
+ host: str | None = None,
+ port: int | None = None,
+ path: str | None = None,
+ query: str | None = None,
+ fragment: str | None = None,
+ ):
+ if path and not path.startswith("/"):
+ path = "/" + path
+ if scheme is not None:
+ scheme = scheme.lower()
+ return super().__new__(cls, scheme, auth, host, port, path, query, fragment)
+
+ @property
+ def hostname(self) -> str | None:
+ """For backwards-compatibility with urlparse. We're nice like that."""
+ return self.host
+
+ @property
+ def request_uri(self) -> str:
+ """Absolute path including the query string."""
+ uri = self.path or "/"
+
+ if self.query is not None:
+ uri += "?" + self.query
+
+ return uri
+
+ @property
+ def authority(self) -> str | None:
+ """
+ Authority component as defined in RFC 3986 3.2.
+ This includes userinfo (auth), host and port.
+
+ i.e.
+ userinfo@host:port
+ """
+ userinfo = self.auth
+ netloc = self.netloc
+ if netloc is None or userinfo is None:
+ return netloc
+ else:
+ return f"{userinfo}@{netloc}"
+
+ @property
+ def netloc(self) -> str | None:
+ """
+ Network location including host and port.
+
+ If you need the equivalent of urllib.parse's ``netloc``,
+ use the ``authority`` property instead.
+ """
+ if self.host is None:
+ return None
+ if self.port:
+ return f"{self.host}:{self.port}"
+ return self.host
+
+ @property
+ def url(self) -> str:
+ """
+ Convert self into a url
+
+ This function should more or less round-trip with :func:`.parse_url`. The
+ returned url may not be exactly the same as the url inputted to
+ :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls
+ with a blank port will have : removed).
+
+ Example:
+
+ .. code-block:: python
+
+ import urllib3
+
+ U = urllib3.util.parse_url("https://google.com/mail/")
+
+ print(U.url)
+ # "https://google.com/mail/"
+
+ print( urllib3.util.Url("https", "username:password",
+ "host.com", 80, "/path", "query", "fragment"
+ ).url
+ )
+ # "https://username:password@host.com:80/path?query#fragment"
+ """
+ scheme, auth, host, port, path, query, fragment = self
+ url = ""
+
+ # We use "is not None" we want things to happen with empty strings (or 0 port)
+ if scheme is not None:
+ url += scheme + "://"
+ if auth is not None:
+ url += auth + "@"
+ if host is not None:
+ url += host
+ if port is not None:
+ url += ":" + str(port)
+ if path is not None:
+ url += path
+ if query is not None:
+ url += "?" + query
+ if fragment is not None:
+ url += "#" + fragment
+
+ return url
+
+ def __str__(self) -> str:
+ return self.url
+
+
+@typing.overload
+def _encode_invalid_chars(
+ component: str, allowed_chars: typing.Container[str]
+) -> str: # Abstract
+ ...
+
+
+@typing.overload
+def _encode_invalid_chars(
+ component: None, allowed_chars: typing.Container[str]
+) -> None: # Abstract
+ ...
+
+
+def _encode_invalid_chars(
+ component: str | None, allowed_chars: typing.Container[str]
+) -> str | None:
+ """Percent-encodes a URI component without reapplying
+ onto an already percent-encoded component.
+ """
+ if component is None:
+ return component
+
+ component = to_str(component)
+
+ # Normalize existing percent-encoded bytes.
+ # Try to see if the component we're encoding is already percent-encoded
+ # so we can skip all '%' characters but still encode all others.
+ component, percent_encodings = _PERCENT_RE.subn(
+ lambda match: match.group(0).upper(), component
+ )
+
+ uri_bytes = component.encode("utf-8", "surrogatepass")
+ is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
+ encoded_component = bytearray()
+
+ for i in range(0, len(uri_bytes)):
+ # Will return a single character bytestring
+ byte = uri_bytes[i : i + 1]
+ byte_ord = ord(byte)
+ if (is_percent_encoded and byte == b"%") or (
+ byte_ord < 128 and byte.decode() in allowed_chars
+ ):
+ encoded_component += byte
+ continue
+ encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper()))
+
+ return encoded_component.decode()
+
+
+def _remove_path_dot_segments(path: str) -> str:
+ # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
+ segments = path.split("/") # Turn the path into a list of segments
+ output = [] # Initialize the variable to use to store output
+
+ for segment in segments:
+ # '.' is the current directory, so ignore it, it is superfluous
+ if segment == ".":
+ continue
+ # Anything other than '..', should be appended to the output
+ if segment != "..":
+ output.append(segment)
+ # In this case segment == '..', if we can, we should pop the last
+ # element
+ elif output:
+ output.pop()
+
+ # If the path starts with '/' and the output is empty or the first string
+ # is non-empty
+ if path.startswith("/") and (not output or output[0]):
+ output.insert(0, "")
+
+ # If the path starts with '/.' or '/..' ensure we add one more empty
+ # string to add a trailing '/'
+ if path.endswith(("/.", "/..")):
+ output.append("")
+
+ return "/".join(output)
+
+
+@typing.overload
+def _normalize_host(host: None, scheme: str | None) -> None: ...
+
+
+@typing.overload
+def _normalize_host(host: str, scheme: str | None) -> str: ...
+
+
+def _normalize_host(host: str | None, scheme: str | None) -> str | None:
+ if host:
+ if scheme in _NORMALIZABLE_SCHEMES:
+ is_ipv6 = _IPV6_ADDRZ_RE.match(host)
+ if is_ipv6:
+ # IPv6 hosts of the form 'a::b%zone' are encoded in a URL as
+ # such per RFC 6874: 'a::b%25zone'. Unquote the ZoneID
+ # separator as necessary to return a valid RFC 4007 scoped IP.
+ match = _ZONE_ID_RE.search(host)
+ if match:
+ start, end = match.span(1)
+ zone_id = host[start:end]
+
+ if zone_id.startswith("%25") and zone_id != "%25":
+ zone_id = zone_id[3:]
+ else:
+ zone_id = zone_id[1:]
+ zone_id = _encode_invalid_chars(zone_id, _UNRESERVED_CHARS)
+ return f"{host[:start].lower()}%{zone_id}{host[end:]}"
+ else:
+ return host.lower()
+ elif not _IPV4_RE.match(host):
+ return to_str(
+ b".".join([_idna_encode(label) for label in host.split(".")]),
+ "ascii",
+ )
+ return host
+
+
+def _idna_encode(name: str) -> bytes:
+ if not name.isascii():
+ try:
+ import idna
+ except ImportError:
+ raise LocationParseError(
+ "Unable to parse URL without the 'idna' module"
+ ) from None
+
+ try:
+ return idna.encode(name.lower(), strict=True, std3_rules=True)
+ except idna.IDNAError:
+ raise LocationParseError(
+ f"Name '{name}' is not a valid IDNA label"
+ ) from None
+
+ return name.lower().encode("ascii")
+
+
+def _encode_target(target: str) -> str:
+ """Percent-encodes a request target so that there are no invalid characters
+
+ Pre-condition for this function is that 'target' must start with '/'.
+ If that is the case then _TARGET_RE will always produce a match.
+ """
+ match = _TARGET_RE.match(target)
+ if not match: # Defensive:
+ raise LocationParseError(f"{target!r} is not a valid request URI")
+
+ path, query = match.groups()
+ encoded_target = _encode_invalid_chars(path, _PATH_CHARS)
+ if query is not None:
+ query = _encode_invalid_chars(query, _QUERY_CHARS)
+ encoded_target += "?" + query
+ return encoded_target
+
+
+def parse_url(url: str) -> Url:
+ """
+ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
+ performed to parse incomplete urls. Fields not provided will be None.
+ This parser is RFC 3986 and RFC 6874 compliant.
+
+ The parser logic and helper functions are based heavily on
+ work done in the ``rfc3986`` module.
+
+ :param str url: URL to parse into a :class:`.Url` namedtuple.
+
+ Partly backwards-compatible with :mod:`urllib.parse`.
+
+ Example:
+
+ .. code-block:: python
+
+ import urllib3
+
+ print( urllib3.util.parse_url('http://google.com/mail/'))
+ # Url(scheme='http', host='google.com', port=None, path='/mail/', ...)
+
+ print( urllib3.util.parse_url('google.com:80'))
+ # Url(scheme=None, host='google.com', port=80, path=None, ...)
+
+ print( urllib3.util.parse_url('/foo?bar'))
+ # Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
+ """
+ if not url:
+ # Empty
+ return Url()
+
+ source_url = url
+ if not _SCHEME_RE.search(url):
+ url = "//" + url
+
+ scheme: str | None
+ authority: str | None
+ auth: str | None
+ host: str | None
+ port: str | None
+ port_int: int | None
+ path: str | None
+ query: str | None
+ fragment: str | None
+
+ try:
+ scheme, authority, path, query, fragment = _URI_RE.match(url).groups() # type: ignore[union-attr]
+ normalize_uri = scheme is None or scheme.lower() in _NORMALIZABLE_SCHEMES
+
+ if scheme:
+ scheme = scheme.lower()
+
+ if authority:
+ auth, _, host_port = authority.rpartition("@")
+ auth = auth or None
+ host, port = _HOST_PORT_RE.match(host_port).groups() # type: ignore[union-attr]
+ if auth and normalize_uri:
+ auth = _encode_invalid_chars(auth, _USERINFO_CHARS)
+ if port == "":
+ port = None
+ else:
+ auth, host, port = None, None, None
+
+ if port is not None:
+ port_int = int(port)
+ if not (0 <= port_int <= 65535):
+ raise LocationParseError(url)
+ else:
+ port_int = None
+
+ host = _normalize_host(host, scheme)
+
+ if normalize_uri and path:
+ path = _remove_path_dot_segments(path)
+ path = _encode_invalid_chars(path, _PATH_CHARS)
+ if normalize_uri and query:
+ query = _encode_invalid_chars(query, _QUERY_CHARS)
+ if normalize_uri and fragment:
+ fragment = _encode_invalid_chars(fragment, _FRAGMENT_CHARS)
+
+ except (ValueError, AttributeError) as e:
+ raise LocationParseError(source_url) from e
+
+ # For the sake of backwards compatibility we put empty
+ # string values for path if there are any defined values
+ # beyond the path in the URL.
+ # TODO: Remove this when we break backwards compatibility.
+ if not path:
+ if query is not None or fragment is not None:
+ path = ""
+ else:
+ path = None
+
+ return Url(
+ scheme=scheme,
+ auth=auth,
+ host=host,
+ port=port_int,
+ path=path,
+ query=query,
+ fragment=fragment,
+ )
diff --git a/phivenv/Lib/site-packages/urllib3/util/util.py b/phivenv/Lib/site-packages/urllib3/util/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..35c77e4025842f548565334a3c04cba90f9283d6
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/util.py
@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import typing
+from types import TracebackType
+
+
+def to_bytes(
+ x: str | bytes, encoding: str | None = None, errors: str | None = None
+) -> bytes:
+ if isinstance(x, bytes):
+ return x
+ elif not isinstance(x, str):
+ raise TypeError(f"not expecting type {type(x).__name__}")
+ if encoding or errors:
+ return x.encode(encoding or "utf-8", errors=errors or "strict")
+ return x.encode()
+
+
+def to_str(
+ x: str | bytes, encoding: str | None = None, errors: str | None = None
+) -> str:
+ if isinstance(x, str):
+ return x
+ elif not isinstance(x, bytes):
+ raise TypeError(f"not expecting type {type(x).__name__}")
+ if encoding or errors:
+ return x.decode(encoding or "utf-8", errors=errors or "strict")
+ return x.decode()
+
+
+def reraise(
+ tp: type[BaseException] | None,
+ value: BaseException,
+ tb: TracebackType | None = None,
+) -> typing.NoReturn:
+ try:
+ if value.__traceback__ is not tb:
+ raise value.with_traceback(tb)
+ raise value
+ finally:
+ value = None # type: ignore[assignment]
+ tb = None
diff --git a/phivenv/Lib/site-packages/urllib3/util/wait.py b/phivenv/Lib/site-packages/urllib3/util/wait.py
new file mode 100644
index 0000000000000000000000000000000000000000..aeca0c7ad5b232eeb1ad9c43d315bd1d74eaed9a
--- /dev/null
+++ b/phivenv/Lib/site-packages/urllib3/util/wait.py
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import select
+import socket
+from functools import partial
+
+__all__ = ["wait_for_read", "wait_for_write"]
+
+
+# How should we wait on sockets?
+#
+# There are two types of APIs you can use for waiting on sockets: the fancy
+# modern stateful APIs like epoll/kqueue, and the older stateless APIs like
+# select/poll. The stateful APIs are more efficient when you have a lots of
+# sockets to keep track of, because you can set them up once and then use them
+# lots of times. But we only ever want to wait on a single socket at a time
+# and don't want to keep track of state, so the stateless APIs are actually
+# more efficient. So we want to use select() or poll().
+#
+# Now, how do we choose between select() and poll()? On traditional Unixes,
+# select() has a strange calling convention that makes it slow, or fail
+# altogether, for high-numbered file descriptors. The point of poll() is to fix
+# that, so on Unixes, we prefer poll().
+#
+# On Windows, there is no poll() (or at least Python doesn't provide a wrapper
+# for it), but that's OK, because on Windows, select() doesn't have this
+# strange calling convention; plain select() works fine.
+#
+# So: on Windows we use select(), and everywhere else we use poll(). We also
+# fall back to select() in case poll() is somehow broken or missing.
+
+
+def select_wait_for_socket(
+ sock: socket.socket,
+ read: bool = False,
+ write: bool = False,
+ timeout: float | None = None,
+) -> bool:
+ if not read and not write:
+ raise RuntimeError("must specify at least one of read=True, write=True")
+ rcheck = []
+ wcheck = []
+ if read:
+ rcheck.append(sock)
+ if write:
+ wcheck.append(sock)
+ # When doing a non-blocking connect, most systems signal success by
+ # marking the socket writable. Windows, though, signals success by marked
+ # it as "exceptional". We paper over the difference by checking the write
+ # sockets for both conditions. (The stdlib selectors module does the same
+ # thing.)
+ fn = partial(select.select, rcheck, wcheck, wcheck)
+ rready, wready, xready = fn(timeout)
+ return bool(rready or wready or xready)
+
+
+def poll_wait_for_socket(
+ sock: socket.socket,
+ read: bool = False,
+ write: bool = False,
+ timeout: float | None = None,
+) -> bool:
+ if not read and not write:
+ raise RuntimeError("must specify at least one of read=True, write=True")
+ mask = 0
+ if read:
+ mask |= select.POLLIN
+ if write:
+ mask |= select.POLLOUT
+ poll_obj = select.poll()
+ poll_obj.register(sock, mask)
+
+ # For some reason, poll() takes timeout in milliseconds
+ def do_poll(t: float | None) -> list[tuple[int, int]]:
+ if t is not None:
+ t *= 1000
+ return poll_obj.poll(t)
+
+ return bool(do_poll(timeout))
+
+
+def _have_working_poll() -> bool:
+ # Apparently some systems have a select.poll that fails as soon as you try
+ # to use it, either due to strange configuration or broken monkeypatching
+ # from libraries like eventlet/greenlet.
+ try:
+ poll_obj = select.poll()
+ poll_obj.poll(0)
+ except (AttributeError, OSError):
+ return False
+ else:
+ return True
+
+
+def wait_for_socket(
+ sock: socket.socket,
+ read: bool = False,
+ write: bool = False,
+ timeout: float | None = None,
+) -> bool:
+ # We delay choosing which implementation to use until the first time we're
+ # called. We could do it at import time, but then we might make the wrong
+ # decision if someone goes wild with monkeypatching select.poll after
+ # we're imported.
+ global wait_for_socket
+ if _have_working_poll():
+ wait_for_socket = poll_wait_for_socket
+ elif hasattr(select, "select"):
+ wait_for_socket = select_wait_for_socket
+ return wait_for_socket(sock, read, write, timeout)
+
+
+def wait_for_read(sock: socket.socket, timeout: float | None = None) -> bool:
+ """Waits for reading to be available on a given socket.
+ Returns True if the socket is readable, or False if the timeout expired.
+ """
+ return wait_for_socket(sock, read=True, timeout=timeout)
+
+
+def wait_for_write(sock: socket.socket, timeout: float | None = None) -> bool:
+ """Waits for writing to be available on a given socket.
+ Returns True if the socket is readable, or False if the timeout expired.
+ """
+ return wait_for_socket(sock, write=True, timeout=timeout)
diff --git a/phivenv/Lib/site-packages/yaml/__init__.py b/phivenv/Lib/site-packages/yaml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ec4f203c7e12941ff807c9def9e35b9ae1ab2bd
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/__init__.py
@@ -0,0 +1,390 @@
+
+from .error import *
+
+from .tokens import *
+from .events import *
+from .nodes import *
+
+from .loader import *
+from .dumper import *
+
+__version__ = '6.0.2'
+try:
+ from .cyaml import *
+ __with_libyaml__ = True
+except ImportError:
+ __with_libyaml__ = False
+
+import io
+
+#------------------------------------------------------------------------------
+# XXX "Warnings control" is now deprecated. Leaving in the API function to not
+# break code that uses it.
+#------------------------------------------------------------------------------
+def warnings(settings=None):
+ if settings is None:
+ return {}
+
+#------------------------------------------------------------------------------
+def scan(stream, Loader=Loader):
+ """
+ Scan a YAML stream and produce scanning tokens.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_token():
+ yield loader.get_token()
+ finally:
+ loader.dispose()
+
+def parse(stream, Loader=Loader):
+ """
+ Parse a YAML stream and produce parsing events.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_event():
+ yield loader.get_event()
+ finally:
+ loader.dispose()
+
+def compose(stream, Loader=Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding representation tree.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_node()
+ finally:
+ loader.dispose()
+
+def compose_all(stream, Loader=Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding representation trees.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_node():
+ yield loader.get_node()
+ finally:
+ loader.dispose()
+
+def load(stream, Loader):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+ """
+ loader = Loader(stream)
+ try:
+ return loader.get_single_data()
+ finally:
+ loader.dispose()
+
+def load_all(stream, Loader):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+ """
+ loader = Loader(stream)
+ try:
+ while loader.check_data():
+ yield loader.get_data()
+ finally:
+ loader.dispose()
+
+def full_load(stream):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+
+ Resolve all tags except those known to be
+ unsafe on untrusted input.
+ """
+ return load(stream, FullLoader)
+
+def full_load_all(stream):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+
+ Resolve all tags except those known to be
+ unsafe on untrusted input.
+ """
+ return load_all(stream, FullLoader)
+
+def safe_load(stream):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+
+ Resolve only basic YAML tags. This is known
+ to be safe for untrusted input.
+ """
+ return load(stream, SafeLoader)
+
+def safe_load_all(stream):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+
+ Resolve only basic YAML tags. This is known
+ to be safe for untrusted input.
+ """
+ return load_all(stream, SafeLoader)
+
+def unsafe_load(stream):
+ """
+ Parse the first YAML document in a stream
+ and produce the corresponding Python object.
+
+ Resolve all tags, even those known to be
+ unsafe on untrusted input.
+ """
+ return load(stream, UnsafeLoader)
+
+def unsafe_load_all(stream):
+ """
+ Parse all YAML documents in a stream
+ and produce corresponding Python objects.
+
+ Resolve all tags, even those known to be
+ unsafe on untrusted input.
+ """
+ return load_all(stream, UnsafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+ """
+ Emit YAML parsing events into a stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ stream = io.StringIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ try:
+ for event in events:
+ dumper.emit(event)
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ """
+ Serialize a sequence of representation trees into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ stream = io.StringIO()
+ else:
+ stream = io.BytesIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end)
+ try:
+ dumper.open()
+ for node in nodes:
+ dumper.serialize(node)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a representation tree into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ getvalue = None
+ if stream is None:
+ if encoding is None:
+ stream = io.StringIO()
+ else:
+ stream = io.BytesIO()
+ getvalue = stream.getvalue
+ dumper = Dumper(stream, default_style=default_style,
+ default_flow_style=default_flow_style,
+ canonical=canonical, indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break,
+ encoding=encoding, version=version, tags=tags,
+ explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys)
+ try:
+ dumper.open()
+ for data in documents:
+ dumper.represent(data)
+ dumper.close()
+ finally:
+ dumper.dispose()
+ if getvalue:
+ return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+ """
+ Serialize a sequence of Python objects into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+ """
+ Serialize a Python object into a YAML stream.
+ Produce only basic YAML tags.
+ If stream is None, return the produced string instead.
+ """
+ return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_implicit_resolver(tag, regexp, first=None,
+ Loader=None, Dumper=Dumper):
+ """
+ Add an implicit scalar detector.
+ If an implicit scalar value matches the given regexp,
+ the corresponding tag is assigned to the scalar.
+ first is a sequence of possible initial characters or None.
+ """
+ if Loader is None:
+ loader.Loader.add_implicit_resolver(tag, regexp, first)
+ loader.FullLoader.add_implicit_resolver(tag, regexp, first)
+ loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first)
+ else:
+ Loader.add_implicit_resolver(tag, regexp, first)
+ Dumper.add_implicit_resolver(tag, regexp, first)
+
+def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper):
+ """
+ Add a path based resolver for the given tag.
+ A path is a list of keys that forms a path
+ to a node in the representation tree.
+ Keys can be string values, integers, or None.
+ """
+ if Loader is None:
+ loader.Loader.add_path_resolver(tag, path, kind)
+ loader.FullLoader.add_path_resolver(tag, path, kind)
+ loader.UnsafeLoader.add_path_resolver(tag, path, kind)
+ else:
+ Loader.add_path_resolver(tag, path, kind)
+ Dumper.add_path_resolver(tag, path, kind)
+
+def add_constructor(tag, constructor, Loader=None):
+ """
+ Add a constructor for the given tag.
+ Constructor is a function that accepts a Loader instance
+ and a node object and produces the corresponding Python object.
+ """
+ if Loader is None:
+ loader.Loader.add_constructor(tag, constructor)
+ loader.FullLoader.add_constructor(tag, constructor)
+ loader.UnsafeLoader.add_constructor(tag, constructor)
+ else:
+ Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=None):
+ """
+ Add a multi-constructor for the given tag prefix.
+ Multi-constructor is called for a node if its tag starts with tag_prefix.
+ Multi-constructor accepts a Loader instance, a tag suffix,
+ and a node object and produces the corresponding Python object.
+ """
+ if Loader is None:
+ loader.Loader.add_multi_constructor(tag_prefix, multi_constructor)
+ loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor)
+ loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor)
+ else:
+ Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+def add_representer(data_type, representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Representer is a function accepting a Dumper instance
+ and an instance of the given data type
+ and producing the corresponding representation node.
+ """
+ Dumper.add_representer(data_type, representer)
+
+def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
+ """
+ Add a representer for the given type.
+ Multi-representer is a function accepting a Dumper instance
+ and an instance of the given data type or subtype
+ and producing the corresponding representation node.
+ """
+ Dumper.add_multi_representer(data_type, multi_representer)
+
+class YAMLObjectMetaclass(type):
+ """
+ The metaclass for YAMLObject.
+ """
+ def __init__(cls, name, bases, kwds):
+ super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+ if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+ if isinstance(cls.yaml_loader, list):
+ for loader in cls.yaml_loader:
+ loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+ else:
+ cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+
+ cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(metaclass=YAMLObjectMetaclass):
+ """
+ An object that can dump itself to a YAML stream
+ and load itself from a YAML stream.
+ """
+
+ __slots__ = () # no direct instantiation, so allow immutable subclasses
+
+ yaml_loader = [Loader, FullLoader, UnsafeLoader]
+ yaml_dumper = Dumper
+
+ yaml_tag = None
+ yaml_flow_style = None
+
+ @classmethod
+ def from_yaml(cls, loader, node):
+ """
+ Convert a representation node to a Python object.
+ """
+ return loader.construct_yaml_object(node, cls)
+
+ @classmethod
+ def to_yaml(cls, dumper, data):
+ """
+ Convert a Python object to a representation node.
+ """
+ return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+ flow_style=cls.yaml_flow_style)
+
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/__init__.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f6b9db040a4c0f94b749398551c81232c32fc13d
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/__init__.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/composer.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/composer.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..87c3dc141b402789b030a947697b3eee87aa6b47
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/composer.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/constructor.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/constructor.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b7f8b9e4bba572d1657a99316a22b640d5d06ad4
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/constructor.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/cyaml.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/cyaml.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6b67e9638e771e0e3d19ca6f6bc860674b1aacd5
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/cyaml.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/dumper.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/dumper.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9e9f7cf10a19b0f83e335820e2077664748e6470
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/dumper.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/emitter.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/emitter.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b57701a948a0f5741fa70ca063f2d1464ecb8439
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/emitter.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/error.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/error.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8446e81dbef871459c56608209ece26290279b62
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/error.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/events.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/events.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6c791fd18f0bbe19d2d1f865e4cfc105654f4aa4
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/events.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/loader.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/loader.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f3e6dd1225acb60eb554a59f3db2d3f82ec64513
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/loader.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/nodes.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/nodes.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5fe2c275d7e907d3a7f2b2a4a4e4786a7dc44c6c
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/nodes.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/parser.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/parser.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9593682e12ca55c5e05d1d342b33b0986c5bab75
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/parser.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/reader.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/reader.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..397cf6885e333c68bb0fddd3fd433980c8d6b08a
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/reader.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/representer.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/representer.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7c1a392415201d74ba03c427d4b71278933e9964
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/representer.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/resolver.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/resolver.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..791364f0735bb3086fcced0169dadf180b697baf
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/resolver.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/scanner.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/scanner.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a19bbca8f183f7226106e144fced74ecf5839b25
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/scanner.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/serializer.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/serializer.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..80a4c98bd7d80dd2d3dbf3b3640612986726b5a5
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/serializer.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/__pycache__/tokens.cpython-39.pyc b/phivenv/Lib/site-packages/yaml/__pycache__/tokens.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0bcf8ad387fc91df1bd45856d521d7f869fcc504
Binary files /dev/null and b/phivenv/Lib/site-packages/yaml/__pycache__/tokens.cpython-39.pyc differ
diff --git a/phivenv/Lib/site-packages/yaml/_yaml.cp39-win_amd64.pyd b/phivenv/Lib/site-packages/yaml/_yaml.cp39-win_amd64.pyd
new file mode 100644
index 0000000000000000000000000000000000000000..e2a9f9ea7849e96eb80c6198d7c6a35c18f47a35
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/_yaml.cp39-win_amd64.pyd
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14450ca425002aa878c1ed077ec2197482aaad3a9876ec5e7272fe412361c205
+size 272896
diff --git a/phivenv/Lib/site-packages/yaml/composer.py b/phivenv/Lib/site-packages/yaml/composer.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d15cb40e3b4198819c91c6f8d8b32807fcf53b2
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/composer.py
@@ -0,0 +1,139 @@
+
+__all__ = ['Composer', 'ComposerError']
+
+from .error import MarkedYAMLError
+from .events import *
+from .nodes import *
+
+class ComposerError(MarkedYAMLError):
+ pass
+
+class Composer:
+
+ def __init__(self):
+ self.anchors = {}
+
+ def check_node(self):
+ # Drop the STREAM-START event.
+ if self.check_event(StreamStartEvent):
+ self.get_event()
+
+ # If there are more documents available?
+ return not self.check_event(StreamEndEvent)
+
+ def get_node(self):
+ # Get the root node of the next document.
+ if not self.check_event(StreamEndEvent):
+ return self.compose_document()
+
+ def get_single_node(self):
+ # Drop the STREAM-START event.
+ self.get_event()
+
+ # Compose a document if the stream is not empty.
+ document = None
+ if not self.check_event(StreamEndEvent):
+ document = self.compose_document()
+
+ # Ensure that the stream contains no more documents.
+ if not self.check_event(StreamEndEvent):
+ event = self.get_event()
+ raise ComposerError("expected a single document in the stream",
+ document.start_mark, "but found another document",
+ event.start_mark)
+
+ # Drop the STREAM-END event.
+ self.get_event()
+
+ return document
+
+ def compose_document(self):
+ # Drop the DOCUMENT-START event.
+ self.get_event()
+
+ # Compose the root node.
+ node = self.compose_node(None, None)
+
+ # Drop the DOCUMENT-END event.
+ self.get_event()
+
+ self.anchors = {}
+ return node
+
+ def compose_node(self, parent, index):
+ if self.check_event(AliasEvent):
+ event = self.get_event()
+ anchor = event.anchor
+ if anchor not in self.anchors:
+ raise ComposerError(None, None, "found undefined alias %r"
+ % anchor, event.start_mark)
+ return self.anchors[anchor]
+ event = self.peek_event()
+ anchor = event.anchor
+ if anchor is not None:
+ if anchor in self.anchors:
+ raise ComposerError("found duplicate anchor %r; first occurrence"
+ % anchor, self.anchors[anchor].start_mark,
+ "second occurrence", event.start_mark)
+ self.descend_resolver(parent, index)
+ if self.check_event(ScalarEvent):
+ node = self.compose_scalar_node(anchor)
+ elif self.check_event(SequenceStartEvent):
+ node = self.compose_sequence_node(anchor)
+ elif self.check_event(MappingStartEvent):
+ node = self.compose_mapping_node(anchor)
+ self.ascend_resolver()
+ return node
+
+ def compose_scalar_node(self, anchor):
+ event = self.get_event()
+ tag = event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(ScalarNode, event.value, event.implicit)
+ node = ScalarNode(tag, event.value,
+ event.start_mark, event.end_mark, style=event.style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ return node
+
+ def compose_sequence_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(SequenceNode, None, start_event.implicit)
+ node = SequenceNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ index = 0
+ while not self.check_event(SequenceEndEvent):
+ node.value.append(self.compose_node(node, index))
+ index += 1
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
+ def compose_mapping_node(self, anchor):
+ start_event = self.get_event()
+ tag = start_event.tag
+ if tag is None or tag == '!':
+ tag = self.resolve(MappingNode, None, start_event.implicit)
+ node = MappingNode(tag, [],
+ start_event.start_mark, None,
+ flow_style=start_event.flow_style)
+ if anchor is not None:
+ self.anchors[anchor] = node
+ while not self.check_event(MappingEndEvent):
+ #key_event = self.peek_event()
+ item_key = self.compose_node(node, None)
+ #if item_key in node.value:
+ # raise ComposerError("while composing a mapping", start_event.start_mark,
+ # "found duplicate key", key_event.start_mark)
+ item_value = self.compose_node(node, item_key)
+ #node.value[item_key] = item_value
+ node.value.append((item_key, item_value))
+ end_event = self.get_event()
+ node.end_mark = end_event.end_mark
+ return node
+
diff --git a/phivenv/Lib/site-packages/yaml/constructor.py b/phivenv/Lib/site-packages/yaml/constructor.py
new file mode 100644
index 0000000000000000000000000000000000000000..619acd3070a4845c653fcf22a626e05158035bc2
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/constructor.py
@@ -0,0 +1,748 @@
+
+__all__ = [
+ 'BaseConstructor',
+ 'SafeConstructor',
+ 'FullConstructor',
+ 'UnsafeConstructor',
+ 'Constructor',
+ 'ConstructorError'
+]
+
+from .error import *
+from .nodes import *
+
+import collections.abc, datetime, base64, binascii, re, sys, types
+
+class ConstructorError(MarkedYAMLError):
+ pass
+
+class BaseConstructor:
+
+ yaml_constructors = {}
+ yaml_multi_constructors = {}
+
+ def __init__(self):
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.state_generators = []
+ self.deep_construct = False
+
+ def check_data(self):
+ # If there are more documents available?
+ return self.check_node()
+
+ def check_state_key(self, key):
+ """Block special attributes/methods from being set in a newly created
+ object, to prevent user-controlled methods from being called during
+ deserialization"""
+ if self.get_state_keys_blacklist_regexp().match(key):
+ raise ConstructorError(None, None,
+ "blacklisted key '%s' in instance state found" % (key,), None)
+
+ def get_data(self):
+ # Construct and return the next document.
+ if self.check_node():
+ return self.construct_document(self.get_node())
+
+ def get_single_data(self):
+ # Ensure that the stream contains a single document and construct it.
+ node = self.get_single_node()
+ if node is not None:
+ return self.construct_document(node)
+ return None
+
+ def construct_document(self, node):
+ data = self.construct_object(node)
+ while self.state_generators:
+ state_generators = self.state_generators
+ self.state_generators = []
+ for generator in state_generators:
+ for dummy in generator:
+ pass
+ self.constructed_objects = {}
+ self.recursive_objects = {}
+ self.deep_construct = False
+ return data
+
+ def construct_object(self, node, deep=False):
+ if node in self.constructed_objects:
+ return self.constructed_objects[node]
+ if deep:
+ old_deep = self.deep_construct
+ self.deep_construct = True
+ if node in self.recursive_objects:
+ raise ConstructorError(None, None,
+ "found unconstructable recursive node", node.start_mark)
+ self.recursive_objects[node] = None
+ constructor = None
+ tag_suffix = None
+ if node.tag in self.yaml_constructors:
+ constructor = self.yaml_constructors[node.tag]
+ else:
+ for tag_prefix in self.yaml_multi_constructors:
+ if tag_prefix is not None and node.tag.startswith(tag_prefix):
+ tag_suffix = node.tag[len(tag_prefix):]
+ constructor = self.yaml_multi_constructors[tag_prefix]
+ break
+ else:
+ if None in self.yaml_multi_constructors:
+ tag_suffix = node.tag
+ constructor = self.yaml_multi_constructors[None]
+ elif None in self.yaml_constructors:
+ constructor = self.yaml_constructors[None]
+ elif isinstance(node, ScalarNode):
+ constructor = self.__class__.construct_scalar
+ elif isinstance(node, SequenceNode):
+ constructor = self.__class__.construct_sequence
+ elif isinstance(node, MappingNode):
+ constructor = self.__class__.construct_mapping
+ if tag_suffix is None:
+ data = constructor(self, node)
+ else:
+ data = constructor(self, tag_suffix, node)
+ if isinstance(data, types.GeneratorType):
+ generator = data
+ data = next(generator)
+ if self.deep_construct:
+ for dummy in generator:
+ pass
+ else:
+ self.state_generators.append(generator)
+ self.constructed_objects[node] = data
+ del self.recursive_objects[node]
+ if deep:
+ self.deep_construct = old_deep
+ return data
+
+ def construct_scalar(self, node):
+ if not isinstance(node, ScalarNode):
+ raise ConstructorError(None, None,
+ "expected a scalar node, but found %s" % node.id,
+ node.start_mark)
+ return node.value
+
+ def construct_sequence(self, node, deep=False):
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError(None, None,
+ "expected a sequence node, but found %s" % node.id,
+ node.start_mark)
+ return [self.construct_object(child, deep=deep)
+ for child in node.value]
+
+ def construct_mapping(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ mapping = {}
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ if not isinstance(key, collections.abc.Hashable):
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "found unhashable key", key_node.start_mark)
+ value = self.construct_object(value_node, deep=deep)
+ mapping[key] = value
+ return mapping
+
+ def construct_pairs(self, node, deep=False):
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ pairs = []
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ value = self.construct_object(value_node, deep=deep)
+ pairs.append((key, value))
+ return pairs
+
+ @classmethod
+ def add_constructor(cls, tag, constructor):
+ if not 'yaml_constructors' in cls.__dict__:
+ cls.yaml_constructors = cls.yaml_constructors.copy()
+ cls.yaml_constructors[tag] = constructor
+
+ @classmethod
+ def add_multi_constructor(cls, tag_prefix, multi_constructor):
+ if not 'yaml_multi_constructors' in cls.__dict__:
+ cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
+ cls.yaml_multi_constructors[tag_prefix] = multi_constructor
+
+class SafeConstructor(BaseConstructor):
+
+ def construct_scalar(self, node):
+ if isinstance(node, MappingNode):
+ for key_node, value_node in node.value:
+ if key_node.tag == 'tag:yaml.org,2002:value':
+ return self.construct_scalar(value_node)
+ return super().construct_scalar(node)
+
+ def flatten_mapping(self, node):
+ merge = []
+ index = 0
+ while index < len(node.value):
+ key_node, value_node = node.value[index]
+ if key_node.tag == 'tag:yaml.org,2002:merge':
+ del node.value[index]
+ if isinstance(value_node, MappingNode):
+ self.flatten_mapping(value_node)
+ merge.extend(value_node.value)
+ elif isinstance(value_node, SequenceNode):
+ submerge = []
+ for subnode in value_node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing a mapping",
+ node.start_mark,
+ "expected a mapping for merging, but found %s"
+ % subnode.id, subnode.start_mark)
+ self.flatten_mapping(subnode)
+ submerge.append(subnode.value)
+ submerge.reverse()
+ for value in submerge:
+ merge.extend(value)
+ else:
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "expected a mapping or list of mappings for merging, but found %s"
+ % value_node.id, value_node.start_mark)
+ elif key_node.tag == 'tag:yaml.org,2002:value':
+ key_node.tag = 'tag:yaml.org,2002:str'
+ index += 1
+ else:
+ index += 1
+ if merge:
+ node.value = merge + node.value
+
+ def construct_mapping(self, node, deep=False):
+ if isinstance(node, MappingNode):
+ self.flatten_mapping(node)
+ return super().construct_mapping(node, deep=deep)
+
+ def construct_yaml_null(self, node):
+ self.construct_scalar(node)
+ return None
+
+ bool_values = {
+ 'yes': True,
+ 'no': False,
+ 'true': True,
+ 'false': False,
+ 'on': True,
+ 'off': False,
+ }
+
+ def construct_yaml_bool(self, node):
+ value = self.construct_scalar(node)
+ return self.bool_values[value.lower()]
+
+ def construct_yaml_int(self, node):
+ value = self.construct_scalar(node)
+ value = value.replace('_', '')
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '0':
+ return 0
+ elif value.startswith('0b'):
+ return sign*int(value[2:], 2)
+ elif value.startswith('0x'):
+ return sign*int(value[2:], 16)
+ elif value[0] == '0':
+ return sign*int(value, 8)
+ elif ':' in value:
+ digits = [int(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*int(value)
+
+ inf_value = 1e300
+ while inf_value != inf_value*inf_value:
+ inf_value *= inf_value
+ nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99).
+
+ def construct_yaml_float(self, node):
+ value = self.construct_scalar(node)
+ value = value.replace('_', '').lower()
+ sign = +1
+ if value[0] == '-':
+ sign = -1
+ if value[0] in '+-':
+ value = value[1:]
+ if value == '.inf':
+ return sign*self.inf_value
+ elif value == '.nan':
+ return self.nan_value
+ elif ':' in value:
+ digits = [float(part) for part in value.split(':')]
+ digits.reverse()
+ base = 1
+ value = 0.0
+ for digit in digits:
+ value += digit*base
+ base *= 60
+ return sign*value
+ else:
+ return sign*float(value)
+
+ def construct_yaml_binary(self, node):
+ try:
+ value = self.construct_scalar(node).encode('ascii')
+ except UnicodeEncodeError as exc:
+ raise ConstructorError(None, None,
+ "failed to convert base64 data into ascii: %s" % exc,
+ node.start_mark)
+ try:
+ if hasattr(base64, 'decodebytes'):
+ return base64.decodebytes(value)
+ else:
+ return base64.decodestring(value)
+ except binascii.Error as exc:
+ raise ConstructorError(None, None,
+ "failed to decode base64 data: %s" % exc, node.start_mark)
+
+ timestamp_regexp = re.compile(
+ r'''^(?P[0-9][0-9][0-9][0-9])
+ -(?P[0-9][0-9]?)
+ -(?P[0-9][0-9]?)
+ (?:(?:[Tt]|[ \t]+)
+ (?P[0-9][0-9]?)
+ :(?P[0-9][0-9])
+ :(?P[0-9][0-9])
+ (?:\.(?P[0-9]*))?
+ (?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?)
+ (?::(?P[0-9][0-9]))?))?)?$''', re.X)
+
+ def construct_yaml_timestamp(self, node):
+ value = self.construct_scalar(node)
+ match = self.timestamp_regexp.match(node.value)
+ values = match.groupdict()
+ year = int(values['year'])
+ month = int(values['month'])
+ day = int(values['day'])
+ if not values['hour']:
+ return datetime.date(year, month, day)
+ hour = int(values['hour'])
+ minute = int(values['minute'])
+ second = int(values['second'])
+ fraction = 0
+ tzinfo = None
+ if values['fraction']:
+ fraction = values['fraction'][:6]
+ while len(fraction) < 6:
+ fraction += '0'
+ fraction = int(fraction)
+ if values['tz_sign']:
+ tz_hour = int(values['tz_hour'])
+ tz_minute = int(values['tz_minute'] or 0)
+ delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
+ if values['tz_sign'] == '-':
+ delta = -delta
+ tzinfo = datetime.timezone(delta)
+ elif values['tz']:
+ tzinfo = datetime.timezone.utc
+ return datetime.datetime(year, month, day, hour, minute, second, fraction,
+ tzinfo=tzinfo)
+
+ def construct_yaml_omap(self, node):
+ # Note: we do not check for duplicate keys, because it's too
+ # CPU-expensive.
+ omap = []
+ yield omap
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing an ordered map", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ omap.append((key, value))
+
+ def construct_yaml_pairs(self, node):
+ # Note: the same code as `construct_yaml_omap`.
+ pairs = []
+ yield pairs
+ if not isinstance(node, SequenceNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a sequence, but found %s" % node.id, node.start_mark)
+ for subnode in node.value:
+ if not isinstance(subnode, MappingNode):
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a mapping of length 1, but found %s" % subnode.id,
+ subnode.start_mark)
+ if len(subnode.value) != 1:
+ raise ConstructorError("while constructing pairs", node.start_mark,
+ "expected a single mapping item, but found %d items" % len(subnode.value),
+ subnode.start_mark)
+ key_node, value_node = subnode.value[0]
+ key = self.construct_object(key_node)
+ value = self.construct_object(value_node)
+ pairs.append((key, value))
+
+ def construct_yaml_set(self, node):
+ data = set()
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_str(self, node):
+ return self.construct_scalar(node)
+
+ def construct_yaml_seq(self, node):
+ data = []
+ yield data
+ data.extend(self.construct_sequence(node))
+
+ def construct_yaml_map(self, node):
+ data = {}
+ yield data
+ value = self.construct_mapping(node)
+ data.update(value)
+
+ def construct_yaml_object(self, node, cls):
+ data = cls.__new__(cls)
+ yield data
+ if hasattr(data, '__setstate__'):
+ state = self.construct_mapping(node, deep=True)
+ data.__setstate__(state)
+ else:
+ state = self.construct_mapping(node)
+ data.__dict__.update(state)
+
+ def construct_undefined(self, node):
+ raise ConstructorError(None, None,
+ "could not determine a constructor for the tag %r" % node.tag,
+ node.start_mark)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:null',
+ SafeConstructor.construct_yaml_null)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:bool',
+ SafeConstructor.construct_yaml_bool)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:int',
+ SafeConstructor.construct_yaml_int)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:float',
+ SafeConstructor.construct_yaml_float)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:binary',
+ SafeConstructor.construct_yaml_binary)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:timestamp',
+ SafeConstructor.construct_yaml_timestamp)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:omap',
+ SafeConstructor.construct_yaml_omap)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:pairs',
+ SafeConstructor.construct_yaml_pairs)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:set',
+ SafeConstructor.construct_yaml_set)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:str',
+ SafeConstructor.construct_yaml_str)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:seq',
+ SafeConstructor.construct_yaml_seq)
+
+SafeConstructor.add_constructor(
+ 'tag:yaml.org,2002:map',
+ SafeConstructor.construct_yaml_map)
+
+SafeConstructor.add_constructor(None,
+ SafeConstructor.construct_undefined)
+
+class FullConstructor(SafeConstructor):
+ # 'extend' is blacklisted because it is used by
+ # construct_python_object_apply to add `listitems` to a newly generate
+ # python instance
+ def get_state_keys_blacklist(self):
+ return ['^extend$', '^__.*__$']
+
+ def get_state_keys_blacklist_regexp(self):
+ if not hasattr(self, 'state_keys_blacklist_regexp'):
+ self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')')
+ return self.state_keys_blacklist_regexp
+
+ def construct_python_str(self, node):
+ return self.construct_scalar(node)
+
+ def construct_python_unicode(self, node):
+ return self.construct_scalar(node)
+
+ def construct_python_bytes(self, node):
+ try:
+ value = self.construct_scalar(node).encode('ascii')
+ except UnicodeEncodeError as exc:
+ raise ConstructorError(None, None,
+ "failed to convert base64 data into ascii: %s" % exc,
+ node.start_mark)
+ try:
+ if hasattr(base64, 'decodebytes'):
+ return base64.decodebytes(value)
+ else:
+ return base64.decodestring(value)
+ except binascii.Error as exc:
+ raise ConstructorError(None, None,
+ "failed to decode base64 data: %s" % exc, node.start_mark)
+
+ def construct_python_long(self, node):
+ return self.construct_yaml_int(node)
+
+ def construct_python_complex(self, node):
+ return complex(self.construct_scalar(node))
+
+ def construct_python_tuple(self, node):
+ return tuple(self.construct_sequence(node))
+
+ def find_python_module(self, name, mark, unsafe=False):
+ if not name:
+ raise ConstructorError("while constructing a Python module", mark,
+ "expected non-empty name appended to the tag", mark)
+ if unsafe:
+ try:
+ __import__(name)
+ except ImportError as exc:
+ raise ConstructorError("while constructing a Python module", mark,
+ "cannot find module %r (%s)" % (name, exc), mark)
+ if name not in sys.modules:
+ raise ConstructorError("while constructing a Python module", mark,
+ "module %r is not imported" % name, mark)
+ return sys.modules[name]
+
+ def find_python_name(self, name, mark, unsafe=False):
+ if not name:
+ raise ConstructorError("while constructing a Python object", mark,
+ "expected non-empty name appended to the tag", mark)
+ if '.' in name:
+ module_name, object_name = name.rsplit('.', 1)
+ else:
+ module_name = 'builtins'
+ object_name = name
+ if unsafe:
+ try:
+ __import__(module_name)
+ except ImportError as exc:
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find module %r (%s)" % (module_name, exc), mark)
+ if module_name not in sys.modules:
+ raise ConstructorError("while constructing a Python object", mark,
+ "module %r is not imported" % module_name, mark)
+ module = sys.modules[module_name]
+ if not hasattr(module, object_name):
+ raise ConstructorError("while constructing a Python object", mark,
+ "cannot find %r in the module %r"
+ % (object_name, module.__name__), mark)
+ return getattr(module, object_name)
+
+ def construct_python_name(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python name", node.start_mark,
+ "expected the empty value, but found %r" % value, node.start_mark)
+ return self.find_python_name(suffix, node.start_mark)
+
+ def construct_python_module(self, suffix, node):
+ value = self.construct_scalar(node)
+ if value:
+ raise ConstructorError("while constructing a Python module", node.start_mark,
+ "expected the empty value, but found %r" % value, node.start_mark)
+ return self.find_python_module(suffix, node.start_mark)
+
+ def make_python_instance(self, suffix, node,
+ args=None, kwds=None, newobj=False, unsafe=False):
+ if not args:
+ args = []
+ if not kwds:
+ kwds = {}
+ cls = self.find_python_name(suffix, node.start_mark)
+ if not (unsafe or isinstance(cls, type)):
+ raise ConstructorError("while constructing a Python instance", node.start_mark,
+ "expected a class, but found %r" % type(cls),
+ node.start_mark)
+ if newobj and isinstance(cls, type):
+ return cls.__new__(cls, *args, **kwds)
+ else:
+ return cls(*args, **kwds)
+
+ def set_python_instance_state(self, instance, state, unsafe=False):
+ if hasattr(instance, '__setstate__'):
+ instance.__setstate__(state)
+ else:
+ slotstate = {}
+ if isinstance(state, tuple) and len(state) == 2:
+ state, slotstate = state
+ if hasattr(instance, '__dict__'):
+ if not unsafe and state:
+ for key in state.keys():
+ self.check_state_key(key)
+ instance.__dict__.update(state)
+ elif state:
+ slotstate.update(state)
+ for key, value in slotstate.items():
+ if not unsafe:
+ self.check_state_key(key)
+ setattr(instance, key, value)
+
+ def construct_python_object(self, suffix, node):
+ # Format:
+ # !!python/object:module.name { ... state ... }
+ instance = self.make_python_instance(suffix, node, newobj=True)
+ yield instance
+ deep = hasattr(instance, '__setstate__')
+ state = self.construct_mapping(node, deep=deep)
+ self.set_python_instance_state(instance, state)
+
+ def construct_python_object_apply(self, suffix, node, newobj=False):
+ # Format:
+ # !!python/object/apply # (or !!python/object/new)
+ # args: [ ... arguments ... ]
+ # kwds: { ... keywords ... }
+ # state: ... state ...
+ # listitems: [ ... listitems ... ]
+ # dictitems: { ... dictitems ... }
+ # or short format:
+ # !!python/object/apply [ ... arguments ... ]
+ # The difference between !!python/object/apply and !!python/object/new
+ # is how an object is created, check make_python_instance for details.
+ if isinstance(node, SequenceNode):
+ args = self.construct_sequence(node, deep=True)
+ kwds = {}
+ state = {}
+ listitems = []
+ dictitems = {}
+ else:
+ value = self.construct_mapping(node, deep=True)
+ args = value.get('args', [])
+ kwds = value.get('kwds', {})
+ state = value.get('state', {})
+ listitems = value.get('listitems', [])
+ dictitems = value.get('dictitems', {})
+ instance = self.make_python_instance(suffix, node, args, kwds, newobj)
+ if state:
+ self.set_python_instance_state(instance, state)
+ if listitems:
+ instance.extend(listitems)
+ if dictitems:
+ for key in dictitems:
+ instance[key] = dictitems[key]
+ return instance
+
+ def construct_python_object_new(self, suffix, node):
+ return self.construct_python_object_apply(suffix, node, newobj=True)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/none',
+ FullConstructor.construct_yaml_null)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/bool',
+ FullConstructor.construct_yaml_bool)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/str',
+ FullConstructor.construct_python_str)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/unicode',
+ FullConstructor.construct_python_unicode)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/bytes',
+ FullConstructor.construct_python_bytes)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/int',
+ FullConstructor.construct_yaml_int)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/long',
+ FullConstructor.construct_python_long)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/float',
+ FullConstructor.construct_yaml_float)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/complex',
+ FullConstructor.construct_python_complex)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/list',
+ FullConstructor.construct_yaml_seq)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/tuple',
+ FullConstructor.construct_python_tuple)
+
+FullConstructor.add_constructor(
+ 'tag:yaml.org,2002:python/dict',
+ FullConstructor.construct_yaml_map)
+
+FullConstructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/name:',
+ FullConstructor.construct_python_name)
+
+class UnsafeConstructor(FullConstructor):
+
+ def find_python_module(self, name, mark):
+ return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True)
+
+ def find_python_name(self, name, mark):
+ return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True)
+
+ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False):
+ return super(UnsafeConstructor, self).make_python_instance(
+ suffix, node, args, kwds, newobj, unsafe=True)
+
+ def set_python_instance_state(self, instance, state):
+ return super(UnsafeConstructor, self).set_python_instance_state(
+ instance, state, unsafe=True)
+
+UnsafeConstructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/module:',
+ UnsafeConstructor.construct_python_module)
+
+UnsafeConstructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object:',
+ UnsafeConstructor.construct_python_object)
+
+UnsafeConstructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object/new:',
+ UnsafeConstructor.construct_python_object_new)
+
+UnsafeConstructor.add_multi_constructor(
+ 'tag:yaml.org,2002:python/object/apply:',
+ UnsafeConstructor.construct_python_object_apply)
+
+# Constructor is same as UnsafeConstructor. Need to leave this in place in case
+# people have extended it directly.
+class Constructor(UnsafeConstructor):
+ pass
diff --git a/phivenv/Lib/site-packages/yaml/cyaml.py b/phivenv/Lib/site-packages/yaml/cyaml.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c21345879b298bb8668201bebe7d289586b17f9
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/cyaml.py
@@ -0,0 +1,101 @@
+
+__all__ = [
+ 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader',
+ 'CBaseDumper', 'CSafeDumper', 'CDumper'
+]
+
+from yaml._yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CFullLoader(CParser, FullConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ FullConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CUnsafeLoader(CParser, UnsafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ UnsafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+ def __init__(self, stream):
+ CParser.__init__(self, stream)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ CEmitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width, encoding=encoding,
+ allow_unicode=allow_unicode, line_break=line_break,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
diff --git a/phivenv/Lib/site-packages/yaml/dumper.py b/phivenv/Lib/site-packages/yaml/dumper.py
new file mode 100644
index 0000000000000000000000000000000000000000..6aadba551f3836b02f4752277f4b3027073defad
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/dumper.py
@@ -0,0 +1,62 @@
+
+__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
+
+from .emitter import *
+from .serializer import *
+from .representer import *
+from .resolver import *
+
+class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
+class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ SafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
+class Dumper(Emitter, Serializer, Representer, Resolver):
+
+ def __init__(self, stream,
+ default_style=None, default_flow_style=False,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None, sort_keys=True):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ Representer.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style, sort_keys=sort_keys)
+ Resolver.__init__(self)
+
diff --git a/phivenv/Lib/site-packages/yaml/emitter.py b/phivenv/Lib/site-packages/yaml/emitter.py
new file mode 100644
index 0000000000000000000000000000000000000000..a664d011162af69184df2f8e59ab7feec818f7c7
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/emitter.py
@@ -0,0 +1,1137 @@
+
+# Emitter expects events obeying the following grammar:
+# stream ::= STREAM-START document* STREAM-END
+# document ::= DOCUMENT-START node DOCUMENT-END
+# node ::= SCALAR | sequence | mapping
+# sequence ::= SEQUENCE-START node* SEQUENCE-END
+# mapping ::= MAPPING-START (node node)* MAPPING-END
+
+__all__ = ['Emitter', 'EmitterError']
+
+from .error import YAMLError
+from .events import *
+
+class EmitterError(YAMLError):
+ pass
+
+class ScalarAnalysis:
+ def __init__(self, scalar, empty, multiline,
+ allow_flow_plain, allow_block_plain,
+ allow_single_quoted, allow_double_quoted,
+ allow_block):
+ self.scalar = scalar
+ self.empty = empty
+ self.multiline = multiline
+ self.allow_flow_plain = allow_flow_plain
+ self.allow_block_plain = allow_block_plain
+ self.allow_single_quoted = allow_single_quoted
+ self.allow_double_quoted = allow_double_quoted
+ self.allow_block = allow_block
+
+class Emitter:
+
+ DEFAULT_TAG_PREFIXES = {
+ '!' : '!',
+ 'tag:yaml.org,2002:' : '!!',
+ }
+
+ def __init__(self, stream, canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None):
+
+ # The stream should have the methods `write` and possibly `flush`.
+ self.stream = stream
+
+ # Encoding can be overridden by STREAM-START.
+ self.encoding = None
+
+ # Emitter is a state machine with a stack of states to handle nested
+ # structures.
+ self.states = []
+ self.state = self.expect_stream_start
+
+ # Current event and the event queue.
+ self.events = []
+ self.event = None
+
+ # The current indentation level and the stack of previous indents.
+ self.indents = []
+ self.indent = None
+
+ # Flow level.
+ self.flow_level = 0
+
+ # Contexts.
+ self.root_context = False
+ self.sequence_context = False
+ self.mapping_context = False
+ self.simple_key_context = False
+
+ # Characteristics of the last emitted character:
+ # - current position.
+ # - is it a whitespace?
+ # - is it an indention character
+ # (indentation space, '-', '?', or ':')?
+ self.line = 0
+ self.column = 0
+ self.whitespace = True
+ self.indention = True
+
+ # Whether the document requires an explicit document indicator
+ self.open_ended = False
+
+ # Formatting details.
+ self.canonical = canonical
+ self.allow_unicode = allow_unicode
+ self.best_indent = 2
+ if indent and 1 < indent < 10:
+ self.best_indent = indent
+ self.best_width = 80
+ if width and width > self.best_indent*2:
+ self.best_width = width
+ self.best_line_break = '\n'
+ if line_break in ['\r', '\n', '\r\n']:
+ self.best_line_break = line_break
+
+ # Tag prefixes.
+ self.tag_prefixes = None
+
+ # Prepared anchor and tag.
+ self.prepared_anchor = None
+ self.prepared_tag = None
+
+ # Scalar analysis and style.
+ self.analysis = None
+ self.style = None
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def emit(self, event):
+ self.events.append(event)
+ while not self.need_more_events():
+ self.event = self.events.pop(0)
+ self.state()
+ self.event = None
+
+ # In some cases, we wait for a few next events before emitting.
+
+ def need_more_events(self):
+ if not self.events:
+ return True
+ event = self.events[0]
+ if isinstance(event, DocumentStartEvent):
+ return self.need_events(1)
+ elif isinstance(event, SequenceStartEvent):
+ return self.need_events(2)
+ elif isinstance(event, MappingStartEvent):
+ return self.need_events(3)
+ else:
+ return False
+
+ def need_events(self, count):
+ level = 0
+ for event in self.events[1:]:
+ if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
+ level += 1
+ elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
+ level -= 1
+ elif isinstance(event, StreamEndEvent):
+ level = -1
+ if level < 0:
+ return False
+ return (len(self.events) < count+1)
+
+ def increase_indent(self, flow=False, indentless=False):
+ self.indents.append(self.indent)
+ if self.indent is None:
+ if flow:
+ self.indent = self.best_indent
+ else:
+ self.indent = 0
+ elif not indentless:
+ self.indent += self.best_indent
+
+ # States.
+
+ # Stream handlers.
+
+ def expect_stream_start(self):
+ if isinstance(self.event, StreamStartEvent):
+ if self.event.encoding and not hasattr(self.stream, 'encoding'):
+ self.encoding = self.event.encoding
+ self.write_stream_start()
+ self.state = self.expect_first_document_start
+ else:
+ raise EmitterError("expected StreamStartEvent, but got %s"
+ % self.event)
+
+ def expect_nothing(self):
+ raise EmitterError("expected nothing, but got %s" % self.event)
+
+ # Document handlers.
+
+ def expect_first_document_start(self):
+ return self.expect_document_start(first=True)
+
+ def expect_document_start(self, first=False):
+ if isinstance(self.event, DocumentStartEvent):
+ if (self.event.version or self.event.tags) and self.open_ended:
+ self.write_indicator('...', True)
+ self.write_indent()
+ if self.event.version:
+ version_text = self.prepare_version(self.event.version)
+ self.write_version_directive(version_text)
+ self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
+ if self.event.tags:
+ handles = sorted(self.event.tags.keys())
+ for handle in handles:
+ prefix = self.event.tags[handle]
+ self.tag_prefixes[prefix] = handle
+ handle_text = self.prepare_tag_handle(handle)
+ prefix_text = self.prepare_tag_prefix(prefix)
+ self.write_tag_directive(handle_text, prefix_text)
+ implicit = (first and not self.event.explicit and not self.canonical
+ and not self.event.version and not self.event.tags
+ and not self.check_empty_document())
+ if not implicit:
+ self.write_indent()
+ self.write_indicator('---', True)
+ if self.canonical:
+ self.write_indent()
+ self.state = self.expect_document_root
+ elif isinstance(self.event, StreamEndEvent):
+ if self.open_ended:
+ self.write_indicator('...', True)
+ self.write_indent()
+ self.write_stream_end()
+ self.state = self.expect_nothing
+ else:
+ raise EmitterError("expected DocumentStartEvent, but got %s"
+ % self.event)
+
+ def expect_document_end(self):
+ if isinstance(self.event, DocumentEndEvent):
+ self.write_indent()
+ if self.event.explicit:
+ self.write_indicator('...', True)
+ self.write_indent()
+ self.flush_stream()
+ self.state = self.expect_document_start
+ else:
+ raise EmitterError("expected DocumentEndEvent, but got %s"
+ % self.event)
+
+ def expect_document_root(self):
+ self.states.append(self.expect_document_end)
+ self.expect_node(root=True)
+
+ # Node handlers.
+
+ def expect_node(self, root=False, sequence=False, mapping=False,
+ simple_key=False):
+ self.root_context = root
+ self.sequence_context = sequence
+ self.mapping_context = mapping
+ self.simple_key_context = simple_key
+ if isinstance(self.event, AliasEvent):
+ self.expect_alias()
+ elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
+ self.process_anchor('&')
+ self.process_tag()
+ if isinstance(self.event, ScalarEvent):
+ self.expect_scalar()
+ elif isinstance(self.event, SequenceStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_sequence():
+ self.expect_flow_sequence()
+ else:
+ self.expect_block_sequence()
+ elif isinstance(self.event, MappingStartEvent):
+ if self.flow_level or self.canonical or self.event.flow_style \
+ or self.check_empty_mapping():
+ self.expect_flow_mapping()
+ else:
+ self.expect_block_mapping()
+ else:
+ raise EmitterError("expected NodeEvent, but got %s" % self.event)
+
+ def expect_alias(self):
+ if self.event.anchor is None:
+ raise EmitterError("anchor is not specified for alias")
+ self.process_anchor('*')
+ self.state = self.states.pop()
+
+ def expect_scalar(self):
+ self.increase_indent(flow=True)
+ self.process_scalar()
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+
+ # Flow sequence handlers.
+
+ def expect_flow_sequence(self):
+ self.write_indicator('[', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_sequence_item
+
+ def expect_first_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator(']', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ def expect_flow_sequence_item(self):
+ if isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(',', False)
+ self.write_indent()
+ self.write_indicator(']', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.states.append(self.expect_flow_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Flow mapping handlers.
+
+ def expect_flow_mapping(self):
+ self.write_indicator('{', True, whitespace=True)
+ self.flow_level += 1
+ self.increase_indent(flow=True)
+ self.state = self.expect_first_flow_mapping_key
+
+ def expect_first_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ self.write_indicator('}', False)
+ self.state = self.states.pop()
+ else:
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_key(self):
+ if isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.flow_level -= 1
+ if self.canonical:
+ self.write_indicator(',', False)
+ self.write_indent()
+ self.write_indicator('}', False)
+ self.state = self.states.pop()
+ else:
+ self.write_indicator(',', False)
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ if not self.canonical and self.check_simple_key():
+ self.states.append(self.expect_flow_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True)
+ self.states.append(self.expect_flow_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_simple_value(self):
+ self.write_indicator(':', False)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_flow_mapping_value(self):
+ if self.canonical or self.column > self.best_width:
+ self.write_indent()
+ self.write_indicator(':', True)
+ self.states.append(self.expect_flow_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Block sequence handlers.
+
+ def expect_block_sequence(self):
+ indentless = (self.mapping_context and not self.indention)
+ self.increase_indent(flow=False, indentless=indentless)
+ self.state = self.expect_first_block_sequence_item
+
+ def expect_first_block_sequence_item(self):
+ return self.expect_block_sequence_item(first=True)
+
+ def expect_block_sequence_item(self, first=False):
+ if not first and isinstance(self.event, SequenceEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ self.write_indicator('-', True, indention=True)
+ self.states.append(self.expect_block_sequence_item)
+ self.expect_node(sequence=True)
+
+ # Block mapping handlers.
+
+ def expect_block_mapping(self):
+ self.increase_indent(flow=False)
+ self.state = self.expect_first_block_mapping_key
+
+ def expect_first_block_mapping_key(self):
+ return self.expect_block_mapping_key(first=True)
+
+ def expect_block_mapping_key(self, first=False):
+ if not first and isinstance(self.event, MappingEndEvent):
+ self.indent = self.indents.pop()
+ self.state = self.states.pop()
+ else:
+ self.write_indent()
+ if self.check_simple_key():
+ self.states.append(self.expect_block_mapping_simple_value)
+ self.expect_node(mapping=True, simple_key=True)
+ else:
+ self.write_indicator('?', True, indention=True)
+ self.states.append(self.expect_block_mapping_value)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_simple_value(self):
+ self.write_indicator(':', False)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ def expect_block_mapping_value(self):
+ self.write_indent()
+ self.write_indicator(':', True, indention=True)
+ self.states.append(self.expect_block_mapping_key)
+ self.expect_node(mapping=True)
+
+ # Checkers.
+
+ def check_empty_sequence(self):
+ return (isinstance(self.event, SequenceStartEvent) and self.events
+ and isinstance(self.events[0], SequenceEndEvent))
+
+ def check_empty_mapping(self):
+ return (isinstance(self.event, MappingStartEvent) and self.events
+ and isinstance(self.events[0], MappingEndEvent))
+
+ def check_empty_document(self):
+ if not isinstance(self.event, DocumentStartEvent) or not self.events:
+ return False
+ event = self.events[0]
+ return (isinstance(event, ScalarEvent) and event.anchor is None
+ and event.tag is None and event.implicit and event.value == '')
+
+ def check_simple_key(self):
+ length = 0
+ if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ length += len(self.prepared_anchor)
+ if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \
+ and self.event.tag is not None:
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(self.event.tag)
+ length += len(self.prepared_tag)
+ if isinstance(self.event, ScalarEvent):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ length += len(self.analysis.scalar)
+ return (length < 128 and (isinstance(self.event, AliasEvent)
+ or (isinstance(self.event, ScalarEvent)
+ and not self.analysis.empty and not self.analysis.multiline)
+ or self.check_empty_sequence() or self.check_empty_mapping()))
+
+ # Anchor, Tag, and Scalar processors.
+
+ def process_anchor(self, indicator):
+ if self.event.anchor is None:
+ self.prepared_anchor = None
+ return
+ if self.prepared_anchor is None:
+ self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+ if self.prepared_anchor:
+ self.write_indicator(indicator+self.prepared_anchor, True)
+ self.prepared_anchor = None
+
+ def process_tag(self):
+ tag = self.event.tag
+ if isinstance(self.event, ScalarEvent):
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ if ((not self.canonical or tag is None) and
+ ((self.style == '' and self.event.implicit[0])
+ or (self.style != '' and self.event.implicit[1]))):
+ self.prepared_tag = None
+ return
+ if self.event.implicit[0] and tag is None:
+ tag = '!'
+ self.prepared_tag = None
+ else:
+ if (not self.canonical or tag is None) and self.event.implicit:
+ self.prepared_tag = None
+ return
+ if tag is None:
+ raise EmitterError("tag is not specified")
+ if self.prepared_tag is None:
+ self.prepared_tag = self.prepare_tag(tag)
+ if self.prepared_tag:
+ self.write_indicator(self.prepared_tag, True)
+ self.prepared_tag = None
+
+ def choose_scalar_style(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.event.style == '"' or self.canonical:
+ return '"'
+ if not self.event.style and self.event.implicit[0]:
+ if (not (self.simple_key_context and
+ (self.analysis.empty or self.analysis.multiline))
+ and (self.flow_level and self.analysis.allow_flow_plain
+ or (not self.flow_level and self.analysis.allow_block_plain))):
+ return ''
+ if self.event.style and self.event.style in '|>':
+ if (not self.flow_level and not self.simple_key_context
+ and self.analysis.allow_block):
+ return self.event.style
+ if not self.event.style or self.event.style == '\'':
+ if (self.analysis.allow_single_quoted and
+ not (self.simple_key_context and self.analysis.multiline)):
+ return '\''
+ return '"'
+
+ def process_scalar(self):
+ if self.analysis is None:
+ self.analysis = self.analyze_scalar(self.event.value)
+ if self.style is None:
+ self.style = self.choose_scalar_style()
+ split = (not self.simple_key_context)
+ #if self.analysis.multiline and split \
+ # and (not self.style or self.style in '\'\"'):
+ # self.write_indent()
+ if self.style == '"':
+ self.write_double_quoted(self.analysis.scalar, split)
+ elif self.style == '\'':
+ self.write_single_quoted(self.analysis.scalar, split)
+ elif self.style == '>':
+ self.write_folded(self.analysis.scalar)
+ elif self.style == '|':
+ self.write_literal(self.analysis.scalar)
+ else:
+ self.write_plain(self.analysis.scalar, split)
+ self.analysis = None
+ self.style = None
+
+ # Analyzers.
+
+ def prepare_version(self, version):
+ major, minor = version
+ if major != 1:
+ raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
+ return '%d.%d' % (major, minor)
+
+ def prepare_tag_handle(self, handle):
+ if not handle:
+ raise EmitterError("tag handle must not be empty")
+ if handle[0] != '!' or handle[-1] != '!':
+ raise EmitterError("tag handle must start and end with '!': %r" % handle)
+ for ch in handle[1:-1]:
+ if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_'):
+ raise EmitterError("invalid character %r in the tag handle: %r"
+ % (ch, handle))
+ return handle
+
+ def prepare_tag_prefix(self, prefix):
+ if not prefix:
+ raise EmitterError("tag prefix must not be empty")
+ chunks = []
+ start = end = 0
+ if prefix[0] == '!':
+ end = 1
+ while end < len(prefix):
+ ch = prefix[end]
+ if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?!:@&=+$,_.~*\'()[]':
+ end += 1
+ else:
+ if start < end:
+ chunks.append(prefix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append('%%%02X' % ord(ch))
+ if start < end:
+ chunks.append(prefix[start:end])
+ return ''.join(chunks)
+
+ def prepare_tag(self, tag):
+ if not tag:
+ raise EmitterError("tag must not be empty")
+ if tag == '!':
+ return tag
+ handle = None
+ suffix = tag
+ prefixes = sorted(self.tag_prefixes.keys())
+ for prefix in prefixes:
+ if tag.startswith(prefix) \
+ and (prefix == '!' or len(prefix) < len(tag)):
+ handle = self.tag_prefixes[prefix]
+ suffix = tag[len(prefix):]
+ chunks = []
+ start = end = 0
+ while end < len(suffix):
+ ch = suffix[end]
+ if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?:@&=+$,_.~*\'()[]' \
+ or (ch == '!' and handle != '!'):
+ end += 1
+ else:
+ if start < end:
+ chunks.append(suffix[start:end])
+ start = end = end+1
+ data = ch.encode('utf-8')
+ for ch in data:
+ chunks.append('%%%02X' % ch)
+ if start < end:
+ chunks.append(suffix[start:end])
+ suffix_text = ''.join(chunks)
+ if handle:
+ return '%s%s' % (handle, suffix_text)
+ else:
+ return '!<%s>' % suffix_text
+
+ def prepare_anchor(self, anchor):
+ if not anchor:
+ raise EmitterError("anchor must not be empty")
+ for ch in anchor:
+ if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_'):
+ raise EmitterError("invalid character %r in the anchor: %r"
+ % (ch, anchor))
+ return anchor
+
+ def analyze_scalar(self, scalar):
+
+ # Empty scalar is a special case.
+ if not scalar:
+ return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
+ allow_flow_plain=False, allow_block_plain=True,
+ allow_single_quoted=True, allow_double_quoted=True,
+ allow_block=False)
+
+ # Indicators and special characters.
+ block_indicators = False
+ flow_indicators = False
+ line_breaks = False
+ special_characters = False
+
+ # Important whitespace combinations.
+ leading_space = False
+ leading_break = False
+ trailing_space = False
+ trailing_break = False
+ break_space = False
+ space_break = False
+
+ # Check document indicators.
+ if scalar.startswith('---') or scalar.startswith('...'):
+ block_indicators = True
+ flow_indicators = True
+
+ # First character or preceded by a whitespace.
+ preceded_by_whitespace = True
+
+ # Last character or followed by a whitespace.
+ followed_by_whitespace = (len(scalar) == 1 or
+ scalar[1] in '\0 \t\r\n\x85\u2028\u2029')
+
+ # The previous character is a space.
+ previous_space = False
+
+ # The previous character is a break.
+ previous_break = False
+
+ index = 0
+ while index < len(scalar):
+ ch = scalar[index]
+
+ # Check for indicators.
+ if index == 0:
+ # Leading indicators are special characters.
+ if ch in '#,[]{}&*!|>\'\"%@`':
+ flow_indicators = True
+ block_indicators = True
+ if ch in '?:':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == '-' and followed_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+ else:
+ # Some indicators cannot appear within a scalar as well.
+ if ch in ',?[]{}':
+ flow_indicators = True
+ if ch == ':':
+ flow_indicators = True
+ if followed_by_whitespace:
+ block_indicators = True
+ if ch == '#' and preceded_by_whitespace:
+ flow_indicators = True
+ block_indicators = True
+
+ # Check for line breaks, special, and unicode characters.
+ if ch in '\n\x85\u2028\u2029':
+ line_breaks = True
+ if not (ch == '\n' or '\x20' <= ch <= '\x7E'):
+ if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF'
+ or '\uE000' <= ch <= '\uFFFD'
+ or '\U00010000' <= ch < '\U0010ffff') and ch != '\uFEFF':
+ unicode_characters = True
+ if not self.allow_unicode:
+ special_characters = True
+ else:
+ special_characters = True
+
+ # Detect important whitespace combinations.
+ if ch == ' ':
+ if index == 0:
+ leading_space = True
+ if index == len(scalar)-1:
+ trailing_space = True
+ if previous_break:
+ break_space = True
+ previous_space = True
+ previous_break = False
+ elif ch in '\n\x85\u2028\u2029':
+ if index == 0:
+ leading_break = True
+ if index == len(scalar)-1:
+ trailing_break = True
+ if previous_space:
+ space_break = True
+ previous_space = False
+ previous_break = True
+ else:
+ previous_space = False
+ previous_break = False
+
+ # Prepare for the next character.
+ index += 1
+ preceded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029')
+ followed_by_whitespace = (index+1 >= len(scalar) or
+ scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029')
+
+ # Let's decide what styles are allowed.
+ allow_flow_plain = True
+ allow_block_plain = True
+ allow_single_quoted = True
+ allow_double_quoted = True
+ allow_block = True
+
+ # Leading and trailing whitespaces are bad for plain scalars.
+ if (leading_space or leading_break
+ or trailing_space or trailing_break):
+ allow_flow_plain = allow_block_plain = False
+
+ # We do not permit trailing spaces for block scalars.
+ if trailing_space:
+ allow_block = False
+
+ # Spaces at the beginning of a new line are only acceptable for block
+ # scalars.
+ if break_space:
+ allow_flow_plain = allow_block_plain = allow_single_quoted = False
+
+ # Spaces followed by breaks, as well as special character are only
+ # allowed for double quoted scalars.
+ if space_break or special_characters:
+ allow_flow_plain = allow_block_plain = \
+ allow_single_quoted = allow_block = False
+
+ # Although the plain scalar writer supports breaks, we never emit
+ # multiline plain scalars.
+ if line_breaks:
+ allow_flow_plain = allow_block_plain = False
+
+ # Flow indicators are forbidden for flow plain scalars.
+ if flow_indicators:
+ allow_flow_plain = False
+
+ # Block indicators are forbidden for block plain scalars.
+ if block_indicators:
+ allow_block_plain = False
+
+ return ScalarAnalysis(scalar=scalar,
+ empty=False, multiline=line_breaks,
+ allow_flow_plain=allow_flow_plain,
+ allow_block_plain=allow_block_plain,
+ allow_single_quoted=allow_single_quoted,
+ allow_double_quoted=allow_double_quoted,
+ allow_block=allow_block)
+
+ # Writers.
+
+ def flush_stream(self):
+ if hasattr(self.stream, 'flush'):
+ self.stream.flush()
+
+ def write_stream_start(self):
+ # Write BOM if needed.
+ if self.encoding and self.encoding.startswith('utf-16'):
+ self.stream.write('\uFEFF'.encode(self.encoding))
+
+ def write_stream_end(self):
+ self.flush_stream()
+
+ def write_indicator(self, indicator, need_whitespace,
+ whitespace=False, indention=False):
+ if self.whitespace or not need_whitespace:
+ data = indicator
+ else:
+ data = ' '+indicator
+ self.whitespace = whitespace
+ self.indention = self.indention and indention
+ self.column += len(data)
+ self.open_ended = False
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_indent(self):
+ indent = self.indent or 0
+ if not self.indention or self.column > indent \
+ or (self.column == indent and not self.whitespace):
+ self.write_line_break()
+ if self.column < indent:
+ self.whitespace = True
+ data = ' '*(indent-self.column)
+ self.column = indent
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_line_break(self, data=None):
+ if data is None:
+ data = self.best_line_break
+ self.whitespace = True
+ self.indention = True
+ self.line += 1
+ self.column = 0
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+
+ def write_version_directive(self, version_text):
+ data = '%%YAML %s' % version_text
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ def write_tag_directive(self, handle_text, prefix_text):
+ data = '%%TAG %s %s' % (handle_text, prefix_text)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_line_break()
+
+ # Scalar streams.
+
+ def write_single_quoted(self, text, split=True):
+ self.write_indicator('\'', True)
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch is None or ch != ' ':
+ if start+1 == end and self.column > self.best_width and split \
+ and start != 0 and end != len(text):
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ if text[start] == '\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'':
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch == '\'':
+ data = '\'\''
+ self.column += 2
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end + 1
+ if ch is not None:
+ spaces = (ch == ' ')
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
+ self.write_indicator('\'', False)
+
+ ESCAPE_REPLACEMENTS = {
+ '\0': '0',
+ '\x07': 'a',
+ '\x08': 'b',
+ '\x09': 't',
+ '\x0A': 'n',
+ '\x0B': 'v',
+ '\x0C': 'f',
+ '\x0D': 'r',
+ '\x1B': 'e',
+ '\"': '\"',
+ '\\': '\\',
+ '\x85': 'N',
+ '\xA0': '_',
+ '\u2028': 'L',
+ '\u2029': 'P',
+ }
+
+ def write_double_quoted(self, text, split=True):
+ self.write_indicator('"', True)
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \
+ or not ('\x20' <= ch <= '\x7E'
+ or (self.allow_unicode
+ and ('\xA0' <= ch <= '\uD7FF'
+ or '\uE000' <= ch <= '\uFFFD'))):
+ if start < end:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ if ch in self.ESCAPE_REPLACEMENTS:
+ data = '\\'+self.ESCAPE_REPLACEMENTS[ch]
+ elif ch <= '\xFF':
+ data = '\\x%02X' % ord(ch)
+ elif ch <= '\uFFFF':
+ data = '\\u%04X' % ord(ch)
+ else:
+ data = '\\U%08X' % ord(ch)
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end+1
+ if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \
+ and self.column+(end-start) > self.best_width and split:
+ data = text[start:end]+'\\'
+ if start < end:
+ start = end
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ if text[start] == ' ':
+ data = '\\'
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ end += 1
+ self.write_indicator('"', False)
+
+ def determine_block_hints(self, text):
+ hints = ''
+ if text:
+ if text[0] in ' \n\x85\u2028\u2029':
+ hints += str(self.best_indent)
+ if text[-1] not in '\n\x85\u2028\u2029':
+ hints += '-'
+ elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029':
+ hints += '+'
+ return hints
+
+ def write_folded(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator('>'+hints, True)
+ if hints[-1:] == '+':
+ self.open_ended = True
+ self.write_line_break()
+ leading_space = True
+ spaces = False
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ if not leading_space and ch is not None and ch != ' ' \
+ and text[start] == '\n':
+ self.write_line_break()
+ leading_space = (ch == ' ')
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ elif spaces:
+ if ch != ' ':
+ if start+1 == end and self.column > self.best_width:
+ self.write_indent()
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in '\n\x85\u2028\u2029')
+ spaces = (ch == ' ')
+ end += 1
+
+ def write_literal(self, text):
+ hints = self.determine_block_hints(text)
+ self.write_indicator('|'+hints, True)
+ if hints[-1:] == '+':
+ self.open_ended = True
+ self.write_line_break()
+ breaks = True
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if breaks:
+ if ch is None or ch not in '\n\x85\u2028\u2029':
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ if ch is not None:
+ self.write_indent()
+ start = end
+ else:
+ if ch is None or ch in '\n\x85\u2028\u2029':
+ data = text[start:end]
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ if ch is None:
+ self.write_line_break()
+ start = end
+ if ch is not None:
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
+
+ def write_plain(self, text, split=True):
+ if self.root_context:
+ self.open_ended = True
+ if not text:
+ return
+ if not self.whitespace:
+ data = ' '
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ self.whitespace = False
+ self.indention = False
+ spaces = False
+ breaks = False
+ start = end = 0
+ while end <= len(text):
+ ch = None
+ if end < len(text):
+ ch = text[end]
+ if spaces:
+ if ch != ' ':
+ if start+1 == end and self.column > self.best_width and split:
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ else:
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ elif breaks:
+ if ch not in '\n\x85\u2028\u2029':
+ if text[start] == '\n':
+ self.write_line_break()
+ for br in text[start:end]:
+ if br == '\n':
+ self.write_line_break()
+ else:
+ self.write_line_break(br)
+ self.write_indent()
+ self.whitespace = False
+ self.indention = False
+ start = end
+ else:
+ if ch is None or ch in ' \n\x85\u2028\u2029':
+ data = text[start:end]
+ self.column += len(data)
+ if self.encoding:
+ data = data.encode(self.encoding)
+ self.stream.write(data)
+ start = end
+ if ch is not None:
+ spaces = (ch == ' ')
+ breaks = (ch in '\n\x85\u2028\u2029')
+ end += 1
diff --git a/phivenv/Lib/site-packages/yaml/error.py b/phivenv/Lib/site-packages/yaml/error.py
new file mode 100644
index 0000000000000000000000000000000000000000..b796b4dc519512c4825ff539a2e6aa20f4d370d0
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/error.py
@@ -0,0 +1,75 @@
+
+__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
+
+class Mark:
+
+ def __init__(self, name, index, line, column, buffer, pointer):
+ self.name = name
+ self.index = index
+ self.line = line
+ self.column = column
+ self.buffer = buffer
+ self.pointer = pointer
+
+ def get_snippet(self, indent=4, max_length=75):
+ if self.buffer is None:
+ return None
+ head = ''
+ start = self.pointer
+ while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029':
+ start -= 1
+ if self.pointer-start > max_length/2-1:
+ head = ' ... '
+ start += 5
+ break
+ tail = ''
+ end = self.pointer
+ while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029':
+ end += 1
+ if end-self.pointer > max_length/2-1:
+ tail = ' ... '
+ end -= 5
+ break
+ snippet = self.buffer[start:end]
+ return ' '*indent + head + snippet + tail + '\n' \
+ + ' '*(indent+self.pointer-start+len(head)) + '^'
+
+ def __str__(self):
+ snippet = self.get_snippet()
+ where = " in \"%s\", line %d, column %d" \
+ % (self.name, self.line+1, self.column+1)
+ if snippet is not None:
+ where += ":\n"+snippet
+ return where
+
+class YAMLError(Exception):
+ pass
+
+class MarkedYAMLError(YAMLError):
+
+ def __init__(self, context=None, context_mark=None,
+ problem=None, problem_mark=None, note=None):
+ self.context = context
+ self.context_mark = context_mark
+ self.problem = problem
+ self.problem_mark = problem_mark
+ self.note = note
+
+ def __str__(self):
+ lines = []
+ if self.context is not None:
+ lines.append(self.context)
+ if self.context_mark is not None \
+ and (self.problem is None or self.problem_mark is None
+ or self.context_mark.name != self.problem_mark.name
+ or self.context_mark.line != self.problem_mark.line
+ or self.context_mark.column != self.problem_mark.column):
+ lines.append(str(self.context_mark))
+ if self.problem is not None:
+ lines.append(self.problem)
+ if self.problem_mark is not None:
+ lines.append(str(self.problem_mark))
+ if self.note is not None:
+ lines.append(self.note)
+ return '\n'.join(lines)
+
diff --git a/phivenv/Lib/site-packages/yaml/events.py b/phivenv/Lib/site-packages/yaml/events.py
new file mode 100644
index 0000000000000000000000000000000000000000..f79ad389cb6c9517e391dcd25534866bc9ccd36a
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/events.py
@@ -0,0 +1,86 @@
+
+# Abstract classes.
+
+class Event(object):
+ def __init__(self, start_mark=None, end_mark=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
+ if hasattr(self, key)]
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+class NodeEvent(Event):
+ def __init__(self, anchor, start_mark=None, end_mark=None):
+ self.anchor = anchor
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class CollectionStartEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
+ flow_style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class CollectionEndEvent(Event):
+ pass
+
+# Implementations.
+
+class StreamStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None, encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndEvent(Event):
+ pass
+
+class DocumentStartEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None, version=None, tags=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+ self.version = version
+ self.tags = tags
+
+class DocumentEndEvent(Event):
+ def __init__(self, start_mark=None, end_mark=None,
+ explicit=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.explicit = explicit
+
+class AliasEvent(NodeEvent):
+ pass
+
+class ScalarEvent(NodeEvent):
+ def __init__(self, anchor, tag, implicit, value,
+ start_mark=None, end_mark=None, style=None):
+ self.anchor = anchor
+ self.tag = tag
+ self.implicit = implicit
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class SequenceStartEvent(CollectionStartEvent):
+ pass
+
+class SequenceEndEvent(CollectionEndEvent):
+ pass
+
+class MappingStartEvent(CollectionStartEvent):
+ pass
+
+class MappingEndEvent(CollectionEndEvent):
+ pass
+
diff --git a/phivenv/Lib/site-packages/yaml/loader.py b/phivenv/Lib/site-packages/yaml/loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..e90c11224c38e559cdf0cb205f0692ebd4fb8681
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/loader.py
@@ -0,0 +1,63 @@
+
+__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader']
+
+from .reader import *
+from .scanner import *
+from .parser import *
+from .composer import *
+from .constructor import *
+from .resolver import *
+
+class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ BaseConstructor.__init__(self)
+ BaseResolver.__init__(self)
+
+class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ FullConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ SafeConstructor.__init__(self)
+ Resolver.__init__(self)
+
+class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
+
+# UnsafeLoader is the same as Loader (which is and was always unsafe on
+# untrusted input). Use of either Loader or UnsafeLoader should be rare, since
+# FullLoad should be able to load almost all YAML safely. Loader is left intact
+# to ensure backwards compatibility.
+class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+ def __init__(self, stream):
+ Reader.__init__(self, stream)
+ Scanner.__init__(self)
+ Parser.__init__(self)
+ Composer.__init__(self)
+ Constructor.__init__(self)
+ Resolver.__init__(self)
diff --git a/phivenv/Lib/site-packages/yaml/nodes.py b/phivenv/Lib/site-packages/yaml/nodes.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4f070c41e1fb1bc01af27d69329e92dded38908
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/nodes.py
@@ -0,0 +1,49 @@
+
+class Node(object):
+ def __init__(self, tag, value, start_mark, end_mark):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ value = self.value
+ #if isinstance(value, list):
+ # if len(value) == 0:
+ # value = ''
+ # elif len(value) == 1:
+ # value = '<1 item>'
+ # else:
+ # value = '<%d items>' % len(value)
+ #else:
+ # if len(value) > 75:
+ # value = repr(value[:70]+u' ... ')
+ # else:
+ # value = repr(value)
+ value = repr(value)
+ return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
+
+class ScalarNode(Node):
+ id = 'scalar'
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
+class CollectionNode(Node):
+ def __init__(self, tag, value,
+ start_mark=None, end_mark=None, flow_style=None):
+ self.tag = tag
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.flow_style = flow_style
+
+class SequenceNode(CollectionNode):
+ id = 'sequence'
+
+class MappingNode(CollectionNode):
+ id = 'mapping'
+
diff --git a/phivenv/Lib/site-packages/yaml/parser.py b/phivenv/Lib/site-packages/yaml/parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..13a5995d292045d0f865a99abf692bd35dc87814
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/parser.py
@@ -0,0 +1,589 @@
+
+# The following YAML grammar is LL(1) and is parsed by a recursive descent
+# parser.
+#
+# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+# implicit_document ::= block_node DOCUMENT-END*
+# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+# block_node_or_indentless_sequence ::=
+# ALIAS
+# | properties (block_content | indentless_block_sequence)?
+# | block_content
+# | indentless_block_sequence
+# block_node ::= ALIAS
+# | properties block_content?
+# | block_content
+# flow_node ::= ALIAS
+# | properties flow_content?
+# | flow_content
+# properties ::= TAG ANCHOR? | ANCHOR TAG?
+# block_content ::= block_collection | flow_collection | SCALAR
+# flow_content ::= flow_collection | SCALAR
+# block_collection ::= block_sequence | block_mapping
+# flow_collection ::= flow_sequence | flow_mapping
+# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+# block_mapping ::= BLOCK-MAPPING_START
+# ((KEY block_node_or_indentless_sequence?)?
+# (VALUE block_node_or_indentless_sequence?)?)*
+# BLOCK-END
+# flow_sequence ::= FLOW-SEQUENCE-START
+# (flow_sequence_entry FLOW-ENTRY)*
+# flow_sequence_entry?
+# FLOW-SEQUENCE-END
+# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+# flow_mapping ::= FLOW-MAPPING-START
+# (flow_mapping_entry FLOW-ENTRY)*
+# flow_mapping_entry?
+# FLOW-MAPPING-END
+# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+#
+# FIRST sets:
+#
+# stream: { STREAM-START }
+# explicit_document: { DIRECTIVE DOCUMENT-START }
+# implicit_document: FIRST(block_node)
+# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_sequence: { BLOCK-SEQUENCE-START }
+# block_mapping: { BLOCK-MAPPING-START }
+# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
+# indentless_sequence: { ENTRY }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_sequence: { FLOW-SEQUENCE-START }
+# flow_mapping: { FLOW-MAPPING-START }
+# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+
+__all__ = ['Parser', 'ParserError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+from .events import *
+from .scanner import *
+
+class ParserError(MarkedYAMLError):
+ pass
+
+class Parser:
+ # Since writing a recursive-descendant parser is a straightforward task, we
+ # do not give many comments here.
+
+ DEFAULT_TAGS = {
+ '!': '!',
+ '!!': 'tag:yaml.org,2002:',
+ }
+
+ def __init__(self):
+ self.current_event = None
+ self.yaml_version = None
+ self.tag_handles = {}
+ self.states = []
+ self.marks = []
+ self.state = self.parse_stream_start
+
+ def dispose(self):
+ # Reset the state attributes (to clear self-references)
+ self.states = []
+ self.state = None
+
+ def check_event(self, *choices):
+ # Check the type of the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ if self.current_event is not None:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.current_event, choice):
+ return True
+ return False
+
+ def peek_event(self):
+ # Get the next event.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ return self.current_event
+
+ def get_event(self):
+ # Get the next event and proceed further.
+ if self.current_event is None:
+ if self.state:
+ self.current_event = self.state()
+ value = self.current_event
+ self.current_event = None
+ return value
+
+ # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END
+ # implicit_document ::= block_node DOCUMENT-END*
+ # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+
+ def parse_stream_start(self):
+
+ # Parse the stream start.
+ token = self.get_token()
+ event = StreamStartEvent(token.start_mark, token.end_mark,
+ encoding=token.encoding)
+
+ # Prepare the next state.
+ self.state = self.parse_implicit_document_start
+
+ return event
+
+ def parse_implicit_document_start(self):
+
+ # Parse an implicit document.
+ if not self.check_token(DirectiveToken, DocumentStartToken,
+ StreamEndToken):
+ self.tag_handles = self.DEFAULT_TAGS
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=False)
+
+ # Prepare the next state.
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_block_node
+
+ return event
+
+ else:
+ return self.parse_document_start()
+
+ def parse_document_start(self):
+
+ # Parse any extra document end indicators.
+ while self.check_token(DocumentEndToken):
+ self.get_token()
+
+ # Parse an explicit document.
+ if not self.check_token(StreamEndToken):
+ token = self.peek_token()
+ start_mark = token.start_mark
+ version, tags = self.process_directives()
+ if not self.check_token(DocumentStartToken):
+ raise ParserError(None, None,
+ "expected '', but found %r"
+ % self.peek_token().id,
+ self.peek_token().start_mark)
+ token = self.get_token()
+ end_mark = token.end_mark
+ event = DocumentStartEvent(start_mark, end_mark,
+ explicit=True, version=version, tags=tags)
+ self.states.append(self.parse_document_end)
+ self.state = self.parse_document_content
+ else:
+ # Parse the end of the stream.
+ token = self.get_token()
+ event = StreamEndEvent(token.start_mark, token.end_mark)
+ assert not self.states
+ assert not self.marks
+ self.state = None
+ return event
+
+ def parse_document_end(self):
+
+ # Parse the document end.
+ token = self.peek_token()
+ start_mark = end_mark = token.start_mark
+ explicit = False
+ if self.check_token(DocumentEndToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ explicit = True
+ event = DocumentEndEvent(start_mark, end_mark,
+ explicit=explicit)
+
+ # Prepare the next state.
+ self.state = self.parse_document_start
+
+ return event
+
+ def parse_document_content(self):
+ if self.check_token(DirectiveToken,
+ DocumentStartToken, DocumentEndToken, StreamEndToken):
+ event = self.process_empty_scalar(self.peek_token().start_mark)
+ self.state = self.states.pop()
+ return event
+ else:
+ return self.parse_block_node()
+
+ def process_directives(self):
+ self.yaml_version = None
+ self.tag_handles = {}
+ while self.check_token(DirectiveToken):
+ token = self.get_token()
+ if token.name == 'YAML':
+ if self.yaml_version is not None:
+ raise ParserError(None, None,
+ "found duplicate YAML directive", token.start_mark)
+ major, minor = token.value
+ if major != 1:
+ raise ParserError(None, None,
+ "found incompatible YAML document (version 1.* is required)",
+ token.start_mark)
+ self.yaml_version = token.value
+ elif token.name == 'TAG':
+ handle, prefix = token.value
+ if handle in self.tag_handles:
+ raise ParserError(None, None,
+ "duplicate tag handle %r" % handle,
+ token.start_mark)
+ self.tag_handles[handle] = prefix
+ if self.tag_handles:
+ value = self.yaml_version, self.tag_handles.copy()
+ else:
+ value = self.yaml_version, None
+ for key in self.DEFAULT_TAGS:
+ if key not in self.tag_handles:
+ self.tag_handles[key] = self.DEFAULT_TAGS[key]
+ return value
+
+ # block_node_or_indentless_sequence ::= ALIAS
+ # | properties (block_content | indentless_block_sequence)?
+ # | block_content
+ # | indentless_block_sequence
+ # block_node ::= ALIAS
+ # | properties block_content?
+ # | block_content
+ # flow_node ::= ALIAS
+ # | properties flow_content?
+ # | flow_content
+ # properties ::= TAG ANCHOR? | ANCHOR TAG?
+ # block_content ::= block_collection | flow_collection | SCALAR
+ # flow_content ::= flow_collection | SCALAR
+ # block_collection ::= block_sequence | block_mapping
+ # flow_collection ::= flow_sequence | flow_mapping
+
+ def parse_block_node(self):
+ return self.parse_node(block=True)
+
+ def parse_flow_node(self):
+ return self.parse_node()
+
+ def parse_block_node_or_indentless_sequence(self):
+ return self.parse_node(block=True, indentless_sequence=True)
+
+ def parse_node(self, block=False, indentless_sequence=False):
+ if self.check_token(AliasToken):
+ token = self.get_token()
+ event = AliasEvent(token.value, token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ else:
+ anchor = None
+ tag = None
+ start_mark = end_mark = tag_mark = None
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ start_mark = token.start_mark
+ end_mark = token.end_mark
+ anchor = token.value
+ if self.check_token(TagToken):
+ token = self.get_token()
+ tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ elif self.check_token(TagToken):
+ token = self.get_token()
+ start_mark = tag_mark = token.start_mark
+ end_mark = token.end_mark
+ tag = token.value
+ if self.check_token(AnchorToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ anchor = token.value
+ if tag is not None:
+ handle, suffix = tag
+ if handle is not None:
+ if handle not in self.tag_handles:
+ raise ParserError("while parsing a node", start_mark,
+ "found undefined tag handle %r" % handle,
+ tag_mark)
+ tag = self.tag_handles[handle]+suffix
+ else:
+ tag = suffix
+ #if tag == '!':
+ # raise ParserError("while parsing a node", start_mark,
+ # "found non-specific tag '!'", tag_mark,
+ # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
+ if start_mark is None:
+ start_mark = end_mark = self.peek_token().start_mark
+ event = None
+ implicit = (tag is None or tag == '!')
+ if indentless_sequence and self.check_token(BlockEntryToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark)
+ self.state = self.parse_indentless_sequence_entry
+ else:
+ if self.check_token(ScalarToken):
+ token = self.get_token()
+ end_mark = token.end_mark
+ if (token.plain and tag is None) or tag == '!':
+ implicit = (True, False)
+ elif tag is None:
+ implicit = (False, True)
+ else:
+ implicit = (False, False)
+ event = ScalarEvent(anchor, tag, implicit, token.value,
+ start_mark, end_mark, style=token.style)
+ self.state = self.states.pop()
+ elif self.check_token(FlowSequenceStartToken):
+ end_mark = self.peek_token().end_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_sequence_first_entry
+ elif self.check_token(FlowMappingStartToken):
+ end_mark = self.peek_token().end_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=True)
+ self.state = self.parse_flow_mapping_first_key
+ elif block and self.check_token(BlockSequenceStartToken):
+ end_mark = self.peek_token().start_mark
+ event = SequenceStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_sequence_first_entry
+ elif block and self.check_token(BlockMappingStartToken):
+ end_mark = self.peek_token().start_mark
+ event = MappingStartEvent(anchor, tag, implicit,
+ start_mark, end_mark, flow_style=False)
+ self.state = self.parse_block_mapping_first_key
+ elif anchor is not None or tag is not None:
+ # Empty scalars are allowed even if a tag or an anchor is
+ # specified.
+ event = ScalarEvent(anchor, tag, (implicit, False), '',
+ start_mark, end_mark)
+ self.state = self.states.pop()
+ else:
+ if block:
+ node = 'block'
+ else:
+ node = 'flow'
+ token = self.peek_token()
+ raise ParserError("while parsing a %s node" % node, start_mark,
+ "expected the node content, but found %r" % token.id,
+ token.start_mark)
+ return event
+
+ # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+
+ def parse_block_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_sequence_entry()
+
+ def parse_block_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken, BlockEndToken):
+ self.states.append(self.parse_block_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_block_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block collection", self.marks[-1],
+ "expected , but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+
+ def parse_indentless_sequence_entry(self):
+ if self.check_token(BlockEntryToken):
+ token = self.get_token()
+ if not self.check_token(BlockEntryToken,
+ KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_indentless_sequence_entry)
+ return self.parse_block_node()
+ else:
+ self.state = self.parse_indentless_sequence_entry
+ return self.process_empty_scalar(token.end_mark)
+ token = self.peek_token()
+ event = SequenceEndEvent(token.start_mark, token.start_mark)
+ self.state = self.states.pop()
+ return event
+
+ # block_mapping ::= BLOCK-MAPPING_START
+ # ((KEY block_node_or_indentless_sequence?)?
+ # (VALUE block_node_or_indentless_sequence?)?)*
+ # BLOCK-END
+
+ def parse_block_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_block_mapping_key()
+
+ def parse_block_mapping_key(self):
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_value)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ if not self.check_token(BlockEndToken):
+ token = self.peek_token()
+ raise ParserError("while parsing a block mapping", self.marks[-1],
+ "expected , but found %r" % token.id, token.start_mark)
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_block_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+ self.states.append(self.parse_block_mapping_key)
+ return self.parse_block_node_or_indentless_sequence()
+ else:
+ self.state = self.parse_block_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_block_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ # flow_sequence ::= FLOW-SEQUENCE-START
+ # (flow_sequence_entry FLOW-ENTRY)*
+ # flow_sequence_entry?
+ # FLOW-SEQUENCE-END
+ # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+ #
+ # Note that while production rules for both flow_sequence_entry and
+ # flow_mapping_entry are equal, their interpretations are different.
+ # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
+ # generate an inline mapping (set syntax).
+
+ def parse_flow_sequence_first_entry(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_sequence_entry(first=True)
+
+ def parse_flow_sequence_entry(self, first=False):
+ if not self.check_token(FlowSequenceEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow sequence", self.marks[-1],
+ "expected ',' or ']', but got %r" % token.id, token.start_mark)
+
+ if self.check_token(KeyToken):
+ token = self.peek_token()
+ event = MappingStartEvent(None, None, True,
+ token.start_mark, token.end_mark,
+ flow_style=True)
+ self.state = self.parse_flow_sequence_entry_mapping_key
+ return event
+ elif not self.check_token(FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = SequenceEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_sequence_entry_mapping_key(self):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+
+ def parse_flow_sequence_entry_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
+ self.states.append(self.parse_flow_sequence_entry_mapping_end)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_sequence_entry_mapping_end
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_sequence_entry_mapping_end(self):
+ self.state = self.parse_flow_sequence_entry
+ token = self.peek_token()
+ return MappingEndEvent(token.start_mark, token.start_mark)
+
+ # flow_mapping ::= FLOW-MAPPING-START
+ # (flow_mapping_entry FLOW-ENTRY)*
+ # flow_mapping_entry?
+ # FLOW-MAPPING-END
+ # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+
+ def parse_flow_mapping_first_key(self):
+ token = self.get_token()
+ self.marks.append(token.start_mark)
+ return self.parse_flow_mapping_key(first=True)
+
+ def parse_flow_mapping_key(self, first=False):
+ if not self.check_token(FlowMappingEndToken):
+ if not first:
+ if self.check_token(FlowEntryToken):
+ self.get_token()
+ else:
+ token = self.peek_token()
+ raise ParserError("while parsing a flow mapping", self.marks[-1],
+ "expected ',' or '}', but got %r" % token.id, token.start_mark)
+ if self.check_token(KeyToken):
+ token = self.get_token()
+ if not self.check_token(ValueToken,
+ FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_value)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_value
+ return self.process_empty_scalar(token.end_mark)
+ elif not self.check_token(FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_empty_value)
+ return self.parse_flow_node()
+ token = self.get_token()
+ event = MappingEndEvent(token.start_mark, token.end_mark)
+ self.state = self.states.pop()
+ self.marks.pop()
+ return event
+
+ def parse_flow_mapping_value(self):
+ if self.check_token(ValueToken):
+ token = self.get_token()
+ if not self.check_token(FlowEntryToken, FlowMappingEndToken):
+ self.states.append(self.parse_flow_mapping_key)
+ return self.parse_flow_node()
+ else:
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(token.end_mark)
+ else:
+ self.state = self.parse_flow_mapping_key
+ token = self.peek_token()
+ return self.process_empty_scalar(token.start_mark)
+
+ def parse_flow_mapping_empty_value(self):
+ self.state = self.parse_flow_mapping_key
+ return self.process_empty_scalar(self.peek_token().start_mark)
+
+ def process_empty_scalar(self, mark):
+ return ScalarEvent(None, None, (True, False), '', mark, mark)
+
diff --git a/phivenv/Lib/site-packages/yaml/reader.py b/phivenv/Lib/site-packages/yaml/reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..774b0219b5932a0ee1c27e637371de5ba8d9cb16
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/reader.py
@@ -0,0 +1,185 @@
+# This module contains abstractions for the input stream. You don't have to
+# looks further, there are no pretty code.
+#
+# We define two classes here.
+#
+# Mark(source, line, column)
+# It's just a record and its only use is producing nice error messages.
+# Parser does not use it for any other purposes.
+#
+# Reader(source, data)
+# Reader determines the encoding of `data` and converts it to unicode.
+# Reader provides the following methods and attributes:
+# reader.peek(length=1) - return the next `length` characters
+# reader.forward(length=1) - move the current position to `length` characters.
+# reader.index - the number of the current character.
+# reader.line, stream.column - the line and the column of the current character.
+
+__all__ = ['Reader', 'ReaderError']
+
+from .error import YAMLError, Mark
+
+import codecs, re
+
+class ReaderError(YAMLError):
+
+ def __init__(self, name, position, character, encoding, reason):
+ self.name = name
+ self.character = character
+ self.position = position
+ self.encoding = encoding
+ self.reason = reason
+
+ def __str__(self):
+ if isinstance(self.character, bytes):
+ return "'%s' codec can't decode byte #x%02x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.encoding, ord(self.character), self.reason,
+ self.name, self.position)
+ else:
+ return "unacceptable character #x%04x: %s\n" \
+ " in \"%s\", position %d" \
+ % (self.character, self.reason,
+ self.name, self.position)
+
+class Reader(object):
+ # Reader:
+ # - determines the data encoding and converts it to a unicode string,
+ # - checks if characters are in allowed range,
+ # - adds '\0' to the end.
+
+ # Reader accepts
+ # - a `bytes` object,
+ # - a `str` object,
+ # - a file-like object with its `read` method returning `str`,
+ # - a file-like object with its `read` method returning `unicode`.
+
+ # Yeah, it's ugly and slow.
+
+ def __init__(self, stream):
+ self.name = None
+ self.stream = None
+ self.stream_pointer = 0
+ self.eof = True
+ self.buffer = ''
+ self.pointer = 0
+ self.raw_buffer = None
+ self.raw_decode = None
+ self.encoding = None
+ self.index = 0
+ self.line = 0
+ self.column = 0
+ if isinstance(stream, str):
+ self.name = ""
+ self.check_printable(stream)
+ self.buffer = stream+'\0'
+ elif isinstance(stream, bytes):
+ self.name = ""
+ self.raw_buffer = stream
+ self.determine_encoding()
+ else:
+ self.stream = stream
+ self.name = getattr(stream, 'name', "")
+ self.eof = False
+ self.raw_buffer = None
+ self.determine_encoding()
+
+ def peek(self, index=0):
+ try:
+ return self.buffer[self.pointer+index]
+ except IndexError:
+ self.update(index+1)
+ return self.buffer[self.pointer+index]
+
+ def prefix(self, length=1):
+ if self.pointer+length >= len(self.buffer):
+ self.update(length)
+ return self.buffer[self.pointer:self.pointer+length]
+
+ def forward(self, length=1):
+ if self.pointer+length+1 >= len(self.buffer):
+ self.update(length+1)
+ while length:
+ ch = self.buffer[self.pointer]
+ self.pointer += 1
+ self.index += 1
+ if ch in '\n\x85\u2028\u2029' \
+ or (ch == '\r' and self.buffer[self.pointer] != '\n'):
+ self.line += 1
+ self.column = 0
+ elif ch != '\uFEFF':
+ self.column += 1
+ length -= 1
+
+ def get_mark(self):
+ if self.stream is None:
+ return Mark(self.name, self.index, self.line, self.column,
+ self.buffer, self.pointer)
+ else:
+ return Mark(self.name, self.index, self.line, self.column,
+ None, None)
+
+ def determine_encoding(self):
+ while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
+ self.update_raw()
+ if isinstance(self.raw_buffer, bytes):
+ if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
+ self.raw_decode = codecs.utf_16_le_decode
+ self.encoding = 'utf-16-le'
+ elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
+ self.raw_decode = codecs.utf_16_be_decode
+ self.encoding = 'utf-16-be'
+ else:
+ self.raw_decode = codecs.utf_8_decode
+ self.encoding = 'utf-8'
+ self.update(1)
+
+ NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]')
+ def check_printable(self, data):
+ match = self.NON_PRINTABLE.search(data)
+ if match:
+ character = match.group()
+ position = self.index+(len(self.buffer)-self.pointer)+match.start()
+ raise ReaderError(self.name, position, ord(character),
+ 'unicode', "special characters are not allowed")
+
+ def update(self, length):
+ if self.raw_buffer is None:
+ return
+ self.buffer = self.buffer[self.pointer:]
+ self.pointer = 0
+ while len(self.buffer) < length:
+ if not self.eof:
+ self.update_raw()
+ if self.raw_decode is not None:
+ try:
+ data, converted = self.raw_decode(self.raw_buffer,
+ 'strict', self.eof)
+ except UnicodeDecodeError as exc:
+ character = self.raw_buffer[exc.start]
+ if self.stream is not None:
+ position = self.stream_pointer-len(self.raw_buffer)+exc.start
+ else:
+ position = exc.start
+ raise ReaderError(self.name, position, character,
+ exc.encoding, exc.reason)
+ else:
+ data = self.raw_buffer
+ converted = len(data)
+ self.check_printable(data)
+ self.buffer += data
+ self.raw_buffer = self.raw_buffer[converted:]
+ if self.eof:
+ self.buffer += '\0'
+ self.raw_buffer = None
+ break
+
+ def update_raw(self, size=4096):
+ data = self.stream.read(size)
+ if self.raw_buffer is None:
+ self.raw_buffer = data
+ else:
+ self.raw_buffer += data
+ self.stream_pointer += len(data)
+ if not data:
+ self.eof = True
diff --git a/phivenv/Lib/site-packages/yaml/representer.py b/phivenv/Lib/site-packages/yaml/representer.py
new file mode 100644
index 0000000000000000000000000000000000000000..808ca06dfbd60c9a23eb079151b74a82ef688749
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/representer.py
@@ -0,0 +1,389 @@
+
+__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
+ 'RepresenterError']
+
+from .error import *
+from .nodes import *
+
+import datetime, copyreg, types, base64, collections
+
+class RepresenterError(YAMLError):
+ pass
+
+class BaseRepresenter:
+
+ yaml_representers = {}
+ yaml_multi_representers = {}
+
+ def __init__(self, default_style=None, default_flow_style=False, sort_keys=True):
+ self.default_style = default_style
+ self.sort_keys = sort_keys
+ self.default_flow_style = default_flow_style
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def represent(self, data):
+ node = self.represent_data(data)
+ self.serialize(node)
+ self.represented_objects = {}
+ self.object_keeper = []
+ self.alias_key = None
+
+ def represent_data(self, data):
+ if self.ignore_aliases(data):
+ self.alias_key = None
+ else:
+ self.alias_key = id(data)
+ if self.alias_key is not None:
+ if self.alias_key in self.represented_objects:
+ node = self.represented_objects[self.alias_key]
+ #if node is None:
+ # raise RepresenterError("recursive objects are not allowed: %r" % data)
+ return node
+ #self.represented_objects[alias_key] = None
+ self.object_keeper.append(data)
+ data_types = type(data).__mro__
+ if data_types[0] in self.yaml_representers:
+ node = self.yaml_representers[data_types[0]](self, data)
+ else:
+ for data_type in data_types:
+ if data_type in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[data_type](self, data)
+ break
+ else:
+ if None in self.yaml_multi_representers:
+ node = self.yaml_multi_representers[None](self, data)
+ elif None in self.yaml_representers:
+ node = self.yaml_representers[None](self, data)
+ else:
+ node = ScalarNode(None, str(data))
+ #if alias_key is not None:
+ # self.represented_objects[alias_key] = node
+ return node
+
+ @classmethod
+ def add_representer(cls, data_type, representer):
+ if not 'yaml_representers' in cls.__dict__:
+ cls.yaml_representers = cls.yaml_representers.copy()
+ cls.yaml_representers[data_type] = representer
+
+ @classmethod
+ def add_multi_representer(cls, data_type, representer):
+ if not 'yaml_multi_representers' in cls.__dict__:
+ cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
+ cls.yaml_multi_representers[data_type] = representer
+
+ def represent_scalar(self, tag, value, style=None):
+ if style is None:
+ style = self.default_style
+ node = ScalarNode(tag, value, style=style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ return node
+
+ def represent_sequence(self, tag, sequence, flow_style=None):
+ value = []
+ node = SequenceNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ for item in sequence:
+ node_item = self.represent_data(item)
+ if not (isinstance(node_item, ScalarNode) and not node_item.style):
+ best_style = False
+ value.append(node_item)
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def represent_mapping(self, tag, mapping, flow_style=None):
+ value = []
+ node = MappingNode(tag, value, flow_style=flow_style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ best_style = True
+ if hasattr(mapping, 'items'):
+ mapping = list(mapping.items())
+ if self.sort_keys:
+ try:
+ mapping = sorted(mapping)
+ except TypeError:
+ pass
+ for item_key, item_value in mapping:
+ node_key = self.represent_data(item_key)
+ node_value = self.represent_data(item_value)
+ if not (isinstance(node_key, ScalarNode) and not node_key.style):
+ best_style = False
+ if not (isinstance(node_value, ScalarNode) and not node_value.style):
+ best_style = False
+ value.append((node_key, node_value))
+ if flow_style is None:
+ if self.default_flow_style is not None:
+ node.flow_style = self.default_flow_style
+ else:
+ node.flow_style = best_style
+ return node
+
+ def ignore_aliases(self, data):
+ return False
+
+class SafeRepresenter(BaseRepresenter):
+
+ def ignore_aliases(self, data):
+ if data is None:
+ return True
+ if isinstance(data, tuple) and data == ():
+ return True
+ if isinstance(data, (str, bytes, bool, int, float)):
+ return True
+
+ def represent_none(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+ def represent_str(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:str', data)
+
+ def represent_binary(self, data):
+ if hasattr(base64, 'encodebytes'):
+ data = base64.encodebytes(data).decode('ascii')
+ else:
+ data = base64.encodestring(data).decode('ascii')
+ return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|')
+
+ def represent_bool(self, data):
+ if data:
+ value = 'true'
+ else:
+ value = 'false'
+ return self.represent_scalar('tag:yaml.org,2002:bool', value)
+
+ def represent_int(self, data):
+ return self.represent_scalar('tag:yaml.org,2002:int', str(data))
+
+ inf_value = 1e300
+ while repr(inf_value) != repr(inf_value*inf_value):
+ inf_value *= inf_value
+
+ def represent_float(self, data):
+ if data != data or (data == 0.0 and data == 1.0):
+ value = '.nan'
+ elif data == self.inf_value:
+ value = '.inf'
+ elif data == -self.inf_value:
+ value = '-.inf'
+ else:
+ value = repr(data).lower()
+ # Note that in some cases `repr(data)` represents a float number
+ # without the decimal parts. For instance:
+ # >>> repr(1e17)
+ # '1e17'
+ # Unfortunately, this is not a valid float representation according
+ # to the definition of the `!!float` tag. We fix this by adding
+ # '.0' before the 'e' symbol.
+ if '.' not in value and 'e' in value:
+ value = value.replace('e', '.0e', 1)
+ return self.represent_scalar('tag:yaml.org,2002:float', value)
+
+ def represent_list(self, data):
+ #pairs = (len(data) > 0 and isinstance(data, list))
+ #if pairs:
+ # for item in data:
+ # if not isinstance(item, tuple) or len(item) != 2:
+ # pairs = False
+ # break
+ #if not pairs:
+ return self.represent_sequence('tag:yaml.org,2002:seq', data)
+ #value = []
+ #for item_key, item_value in data:
+ # value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
+ # [(item_key, item_value)]))
+ #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
+
+ def represent_dict(self, data):
+ return self.represent_mapping('tag:yaml.org,2002:map', data)
+
+ def represent_set(self, data):
+ value = {}
+ for key in data:
+ value[key] = None
+ return self.represent_mapping('tag:yaml.org,2002:set', value)
+
+ def represent_date(self, data):
+ value = data.isoformat()
+ return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+ def represent_datetime(self, data):
+ value = data.isoformat(' ')
+ return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+ def represent_yaml_object(self, tag, data, cls, flow_style=None):
+ if hasattr(data, '__getstate__'):
+ state = data.__getstate__()
+ else:
+ state = data.__dict__.copy()
+ return self.represent_mapping(tag, state, flow_style=flow_style)
+
+ def represent_undefined(self, data):
+ raise RepresenterError("cannot represent an object", data)
+
+SafeRepresenter.add_representer(type(None),
+ SafeRepresenter.represent_none)
+
+SafeRepresenter.add_representer(str,
+ SafeRepresenter.represent_str)
+
+SafeRepresenter.add_representer(bytes,
+ SafeRepresenter.represent_binary)
+
+SafeRepresenter.add_representer(bool,
+ SafeRepresenter.represent_bool)
+
+SafeRepresenter.add_representer(int,
+ SafeRepresenter.represent_int)
+
+SafeRepresenter.add_representer(float,
+ SafeRepresenter.represent_float)
+
+SafeRepresenter.add_representer(list,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(tuple,
+ SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(dict,
+ SafeRepresenter.represent_dict)
+
+SafeRepresenter.add_representer(set,
+ SafeRepresenter.represent_set)
+
+SafeRepresenter.add_representer(datetime.date,
+ SafeRepresenter.represent_date)
+
+SafeRepresenter.add_representer(datetime.datetime,
+ SafeRepresenter.represent_datetime)
+
+SafeRepresenter.add_representer(None,
+ SafeRepresenter.represent_undefined)
+
+class Representer(SafeRepresenter):
+
+ def represent_complex(self, data):
+ if data.imag == 0.0:
+ data = '%r' % data.real
+ elif data.real == 0.0:
+ data = '%rj' % data.imag
+ elif data.imag > 0:
+ data = '%r+%rj' % (data.real, data.imag)
+ else:
+ data = '%r%rj' % (data.real, data.imag)
+ return self.represent_scalar('tag:yaml.org,2002:python/complex', data)
+
+ def represent_tuple(self, data):
+ return self.represent_sequence('tag:yaml.org,2002:python/tuple', data)
+
+ def represent_name(self, data):
+ name = '%s.%s' % (data.__module__, data.__name__)
+ return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '')
+
+ def represent_module(self, data):
+ return self.represent_scalar(
+ 'tag:yaml.org,2002:python/module:'+data.__name__, '')
+
+ def represent_object(self, data):
+ # We use __reduce__ API to save the data. data.__reduce__ returns
+ # a tuple of length 2-5:
+ # (function, args, state, listitems, dictitems)
+
+ # For reconstructing, we calls function(*args), then set its state,
+ # listitems, and dictitems if they are not None.
+
+ # A special case is when function.__name__ == '__newobj__'. In this
+ # case we create the object with args[0].__new__(*args).
+
+ # Another special case is when __reduce__ returns a string - we don't
+ # support it.
+
+ # We produce a !!python/object, !!python/object/new or
+ # !!python/object/apply node.
+
+ cls = type(data)
+ if cls in copyreg.dispatch_table:
+ reduce = copyreg.dispatch_table[cls](data)
+ elif hasattr(data, '__reduce_ex__'):
+ reduce = data.__reduce_ex__(2)
+ elif hasattr(data, '__reduce__'):
+ reduce = data.__reduce__()
+ else:
+ raise RepresenterError("cannot represent an object", data)
+ reduce = (list(reduce)+[None]*5)[:5]
+ function, args, state, listitems, dictitems = reduce
+ args = list(args)
+ if state is None:
+ state = {}
+ if listitems is not None:
+ listitems = list(listitems)
+ if dictitems is not None:
+ dictitems = dict(dictitems)
+ if function.__name__ == '__newobj__':
+ function = args[0]
+ args = args[1:]
+ tag = 'tag:yaml.org,2002:python/object/new:'
+ newobj = True
+ else:
+ tag = 'tag:yaml.org,2002:python/object/apply:'
+ newobj = False
+ function_name = '%s.%s' % (function.__module__, function.__name__)
+ if not args and not listitems and not dictitems \
+ and isinstance(state, dict) and newobj:
+ return self.represent_mapping(
+ 'tag:yaml.org,2002:python/object:'+function_name, state)
+ if not listitems and not dictitems \
+ and isinstance(state, dict) and not state:
+ return self.represent_sequence(tag+function_name, args)
+ value = {}
+ if args:
+ value['args'] = args
+ if state or not isinstance(state, dict):
+ value['state'] = state
+ if listitems:
+ value['listitems'] = listitems
+ if dictitems:
+ value['dictitems'] = dictitems
+ return self.represent_mapping(tag+function_name, value)
+
+ def represent_ordered_dict(self, data):
+ # Provide uniform representation across different Python versions.
+ data_type = type(data)
+ tag = 'tag:yaml.org,2002:python/object/apply:%s.%s' \
+ % (data_type.__module__, data_type.__name__)
+ items = [[key, value] for key, value in data.items()]
+ return self.represent_sequence(tag, [items])
+
+Representer.add_representer(complex,
+ Representer.represent_complex)
+
+Representer.add_representer(tuple,
+ Representer.represent_tuple)
+
+Representer.add_multi_representer(type,
+ Representer.represent_name)
+
+Representer.add_representer(collections.OrderedDict,
+ Representer.represent_ordered_dict)
+
+Representer.add_representer(types.FunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.BuiltinFunctionType,
+ Representer.represent_name)
+
+Representer.add_representer(types.ModuleType,
+ Representer.represent_module)
+
+Representer.add_multi_representer(object,
+ Representer.represent_object)
+
diff --git a/phivenv/Lib/site-packages/yaml/resolver.py b/phivenv/Lib/site-packages/yaml/resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..3522bdaaf6358110b608f4e6503b9d314c82d887
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/resolver.py
@@ -0,0 +1,227 @@
+
+__all__ = ['BaseResolver', 'Resolver']
+
+from .error import *
+from .nodes import *
+
+import re
+
+class ResolverError(YAMLError):
+ pass
+
+class BaseResolver:
+
+ DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str'
+ DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq'
+ DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
+
+ yaml_implicit_resolvers = {}
+ yaml_path_resolvers = {}
+
+ def __init__(self):
+ self.resolver_exact_paths = []
+ self.resolver_prefix_paths = []
+
+ @classmethod
+ def add_implicit_resolver(cls, tag, regexp, first):
+ if not 'yaml_implicit_resolvers' in cls.__dict__:
+ implicit_resolvers = {}
+ for key in cls.yaml_implicit_resolvers:
+ implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:]
+ cls.yaml_implicit_resolvers = implicit_resolvers
+ if first is None:
+ first = [None]
+ for ch in first:
+ cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
+
+ @classmethod
+ def add_path_resolver(cls, tag, path, kind=None):
+ # Note: `add_path_resolver` is experimental. The API could be changed.
+ # `new_path` is a pattern that is matched against the path from the
+ # root to the node that is being considered. `node_path` elements are
+ # tuples `(node_check, index_check)`. `node_check` is a node class:
+ # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
+ # matches any kind of a node. `index_check` could be `None`, a boolean
+ # value, a string value, or a number. `None` and `False` match against
+ # any _value_ of sequence and mapping nodes. `True` matches against
+ # any _key_ of a mapping node. A string `index_check` matches against
+ # a mapping value that corresponds to a scalar key which content is
+ # equal to the `index_check` value. An integer `index_check` matches
+ # against a sequence value with the index equal to `index_check`.
+ if not 'yaml_path_resolvers' in cls.__dict__:
+ cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
+ new_path = []
+ for element in path:
+ if isinstance(element, (list, tuple)):
+ if len(element) == 2:
+ node_check, index_check = element
+ elif len(element) == 1:
+ node_check = element[0]
+ index_check = True
+ else:
+ raise ResolverError("Invalid path element: %s" % element)
+ else:
+ node_check = None
+ index_check = element
+ if node_check is str:
+ node_check = ScalarNode
+ elif node_check is list:
+ node_check = SequenceNode
+ elif node_check is dict:
+ node_check = MappingNode
+ elif node_check not in [ScalarNode, SequenceNode, MappingNode] \
+ and not isinstance(node_check, str) \
+ and node_check is not None:
+ raise ResolverError("Invalid node checker: %s" % node_check)
+ if not isinstance(index_check, (str, int)) \
+ and index_check is not None:
+ raise ResolverError("Invalid index checker: %s" % index_check)
+ new_path.append((node_check, index_check))
+ if kind is str:
+ kind = ScalarNode
+ elif kind is list:
+ kind = SequenceNode
+ elif kind is dict:
+ kind = MappingNode
+ elif kind not in [ScalarNode, SequenceNode, MappingNode] \
+ and kind is not None:
+ raise ResolverError("Invalid node kind: %s" % kind)
+ cls.yaml_path_resolvers[tuple(new_path), kind] = tag
+
+ def descend_resolver(self, current_node, current_index):
+ if not self.yaml_path_resolvers:
+ return
+ exact_paths = {}
+ prefix_paths = []
+ if current_node:
+ depth = len(self.resolver_prefix_paths)
+ for path, kind in self.resolver_prefix_paths[-1]:
+ if self.check_resolver_prefix(depth, path, kind,
+ current_node, current_index):
+ if len(path) > depth:
+ prefix_paths.append((path, kind))
+ else:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ for path, kind in self.yaml_path_resolvers:
+ if not path:
+ exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+ else:
+ prefix_paths.append((path, kind))
+ self.resolver_exact_paths.append(exact_paths)
+ self.resolver_prefix_paths.append(prefix_paths)
+
+ def ascend_resolver(self):
+ if not self.yaml_path_resolvers:
+ return
+ self.resolver_exact_paths.pop()
+ self.resolver_prefix_paths.pop()
+
+ def check_resolver_prefix(self, depth, path, kind,
+ current_node, current_index):
+ node_check, index_check = path[depth-1]
+ if isinstance(node_check, str):
+ if current_node.tag != node_check:
+ return
+ elif node_check is not None:
+ if not isinstance(current_node, node_check):
+ return
+ if index_check is True and current_index is not None:
+ return
+ if (index_check is False or index_check is None) \
+ and current_index is None:
+ return
+ if isinstance(index_check, str):
+ if not (isinstance(current_index, ScalarNode)
+ and index_check == current_index.value):
+ return
+ elif isinstance(index_check, int) and not isinstance(index_check, bool):
+ if index_check != current_index:
+ return
+ return True
+
+ def resolve(self, kind, value, implicit):
+ if kind is ScalarNode and implicit[0]:
+ if value == '':
+ resolvers = self.yaml_implicit_resolvers.get('', [])
+ else:
+ resolvers = self.yaml_implicit_resolvers.get(value[0], [])
+ wildcard_resolvers = self.yaml_implicit_resolvers.get(None, [])
+ for tag, regexp in resolvers + wildcard_resolvers:
+ if regexp.match(value):
+ return tag
+ implicit = implicit[1]
+ if self.yaml_path_resolvers:
+ exact_paths = self.resolver_exact_paths[-1]
+ if kind in exact_paths:
+ return exact_paths[kind]
+ if None in exact_paths:
+ return exact_paths[None]
+ if kind is ScalarNode:
+ return self.DEFAULT_SCALAR_TAG
+ elif kind is SequenceNode:
+ return self.DEFAULT_SEQUENCE_TAG
+ elif kind is MappingNode:
+ return self.DEFAULT_MAPPING_TAG
+
+class Resolver(BaseResolver):
+ pass
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:bool',
+ re.compile(r'''^(?:yes|Yes|YES|no|No|NO
+ |true|True|TRUE|false|False|FALSE
+ |on|On|ON|off|Off|OFF)$''', re.X),
+ list('yYnNtTfFoO'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:float',
+ re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
+ |\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?
+ |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
+ |[-+]?\.(?:inf|Inf|INF)
+ |\.(?:nan|NaN|NAN))$''', re.X),
+ list('-+0123456789.'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:int',
+ re.compile(r'''^(?:[-+]?0b[0-1_]+
+ |[-+]?0[0-7_]+
+ |[-+]?(?:0|[1-9][0-9_]*)
+ |[-+]?0x[0-9a-fA-F_]+
+ |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
+ list('-+0123456789'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:merge',
+ re.compile(r'^(?:<<)$'),
+ ['<'])
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:null',
+ re.compile(r'''^(?: ~
+ |null|Null|NULL
+ | )$''', re.X),
+ ['~', 'n', 'N', ''])
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:timestamp',
+ re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
+ |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
+ (?:[Tt]|[ \t]+)[0-9][0-9]?
+ :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
+ (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
+ list('0123456789'))
+
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:value',
+ re.compile(r'^(?:=)$'),
+ ['='])
+
+# The following resolver is only for documentation purposes. It cannot work
+# because plain scalars cannot start with '!', '&', or '*'.
+Resolver.add_implicit_resolver(
+ 'tag:yaml.org,2002:yaml',
+ re.compile(r'^(?:!|&|\*)$'),
+ list('!&*'))
+
diff --git a/phivenv/Lib/site-packages/yaml/scanner.py b/phivenv/Lib/site-packages/yaml/scanner.py
new file mode 100644
index 0000000000000000000000000000000000000000..de925b07f1eaec33c9c305a8a69f9eb7ac5983c5
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/scanner.py
@@ -0,0 +1,1435 @@
+
+# Scanner produces tokens of the following types:
+# STREAM-START
+# STREAM-END
+# DIRECTIVE(name, value)
+# DOCUMENT-START
+# DOCUMENT-END
+# BLOCK-SEQUENCE-START
+# BLOCK-MAPPING-START
+# BLOCK-END
+# FLOW-SEQUENCE-START
+# FLOW-MAPPING-START
+# FLOW-SEQUENCE-END
+# FLOW-MAPPING-END
+# BLOCK-ENTRY
+# FLOW-ENTRY
+# KEY
+# VALUE
+# ALIAS(value)
+# ANCHOR(value)
+# TAG(value)
+# SCALAR(value, plain, style)
+#
+# Read comments in the Scanner code for more details.
+#
+
+__all__ = ['Scanner', 'ScannerError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+
+class ScannerError(MarkedYAMLError):
+ pass
+
+class SimpleKey:
+ # See below simple keys treatment.
+
+ def __init__(self, token_number, required, index, line, column, mark):
+ self.token_number = token_number
+ self.required = required
+ self.index = index
+ self.line = line
+ self.column = column
+ self.mark = mark
+
+class Scanner:
+
+ def __init__(self):
+ """Initialize the scanner."""
+ # It is assumed that Scanner and Reader will have a common descendant.
+ # Reader do the dirty work of checking for BOM and converting the
+ # input data to Unicode. It also adds NUL to the end.
+ #
+ # Reader supports the following methods
+ # self.peek(i=0) # peek the next i-th character
+ # self.prefix(l=1) # peek the next l characters
+ # self.forward(l=1) # read the next l characters and move the pointer.
+
+ # Had we reached the end of the stream?
+ self.done = False
+
+ # The number of unclosed '{' and '['. `flow_level == 0` means block
+ # context.
+ self.flow_level = 0
+
+ # List of processed tokens that are not yet emitted.
+ self.tokens = []
+
+ # Add the STREAM-START token.
+ self.fetch_stream_start()
+
+ # Number of tokens that were emitted through the `get_token` method.
+ self.tokens_taken = 0
+
+ # The current indentation level.
+ self.indent = -1
+
+ # Past indentation levels.
+ self.indents = []
+
+ # Variables related to simple keys treatment.
+
+ # A simple key is a key that is not denoted by the '?' indicator.
+ # Example of simple keys:
+ # ---
+ # block simple key: value
+ # ? not a simple key:
+ # : { flow simple key: value }
+ # We emit the KEY token before all keys, so when we find a potential
+ # simple key, we try to locate the corresponding ':' indicator.
+ # Simple keys should be limited to a single line and 1024 characters.
+
+ # Can a simple key start at the current position? A simple key may
+ # start:
+ # - at the beginning of the line, not counting indentation spaces
+ # (in block context),
+ # - after '{', '[', ',' (in the flow context),
+ # - after '?', ':', '-' (in the block context).
+ # In the block context, this flag also signifies if a block collection
+ # may start at the current position.
+ self.allow_simple_key = True
+
+ # Keep track of possible simple keys. This is a dictionary. The key
+ # is `flow_level`; there can be no more that one possible simple key
+ # for each level. The value is a SimpleKey record:
+ # (token_number, required, index, line, column, mark)
+ # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
+ # '[', or '{' tokens.
+ self.possible_simple_keys = {}
+
+ # Public methods.
+
+ def check_token(self, *choices):
+ # Check if the next token is one of the given types.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ if not choices:
+ return True
+ for choice in choices:
+ if isinstance(self.tokens[0], choice):
+ return True
+ return False
+
+ def peek_token(self):
+ # Return the next token, but do not delete if from the queue.
+ # Return None if no more tokens.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ return self.tokens[0]
+ else:
+ return None
+
+ def get_token(self):
+ # Return the next token.
+ while self.need_more_tokens():
+ self.fetch_more_tokens()
+ if self.tokens:
+ self.tokens_taken += 1
+ return self.tokens.pop(0)
+
+ # Private methods.
+
+ def need_more_tokens(self):
+ if self.done:
+ return False
+ if not self.tokens:
+ return True
+ # The current token may be a potential simple key, so we
+ # need to look further.
+ self.stale_possible_simple_keys()
+ if self.next_possible_simple_key() == self.tokens_taken:
+ return True
+
+ def fetch_more_tokens(self):
+
+ # Eat whitespaces and comments until we reach the next token.
+ self.scan_to_next_token()
+
+ # Remove obsolete possible simple keys.
+ self.stale_possible_simple_keys()
+
+ # Compare the current indentation and column. It may add some tokens
+ # and decrease the current indentation level.
+ self.unwind_indent(self.column)
+
+ # Peek the next character.
+ ch = self.peek()
+
+ # Is it the end of stream?
+ if ch == '\0':
+ return self.fetch_stream_end()
+
+ # Is it a directive?
+ if ch == '%' and self.check_directive():
+ return self.fetch_directive()
+
+ # Is it the document start?
+ if ch == '-' and self.check_document_start():
+ return self.fetch_document_start()
+
+ # Is it the document end?
+ if ch == '.' and self.check_document_end():
+ return self.fetch_document_end()
+
+ # TODO: support for BOM within a stream.
+ #if ch == '\uFEFF':
+ # return self.fetch_bom() <-- issue BOMToken
+
+ # Note: the order of the following checks is NOT significant.
+
+ # Is it the flow sequence start indicator?
+ if ch == '[':
+ return self.fetch_flow_sequence_start()
+
+ # Is it the flow mapping start indicator?
+ if ch == '{':
+ return self.fetch_flow_mapping_start()
+
+ # Is it the flow sequence end indicator?
+ if ch == ']':
+ return self.fetch_flow_sequence_end()
+
+ # Is it the flow mapping end indicator?
+ if ch == '}':
+ return self.fetch_flow_mapping_end()
+
+ # Is it the flow entry indicator?
+ if ch == ',':
+ return self.fetch_flow_entry()
+
+ # Is it the block entry indicator?
+ if ch == '-' and self.check_block_entry():
+ return self.fetch_block_entry()
+
+ # Is it the key indicator?
+ if ch == '?' and self.check_key():
+ return self.fetch_key()
+
+ # Is it the value indicator?
+ if ch == ':' and self.check_value():
+ return self.fetch_value()
+
+ # Is it an alias?
+ if ch == '*':
+ return self.fetch_alias()
+
+ # Is it an anchor?
+ if ch == '&':
+ return self.fetch_anchor()
+
+ # Is it a tag?
+ if ch == '!':
+ return self.fetch_tag()
+
+ # Is it a literal scalar?
+ if ch == '|' and not self.flow_level:
+ return self.fetch_literal()
+
+ # Is it a folded scalar?
+ if ch == '>' and not self.flow_level:
+ return self.fetch_folded()
+
+ # Is it a single quoted scalar?
+ if ch == '\'':
+ return self.fetch_single()
+
+ # Is it a double quoted scalar?
+ if ch == '\"':
+ return self.fetch_double()
+
+ # It must be a plain scalar then.
+ if self.check_plain():
+ return self.fetch_plain()
+
+ # No? It's an error. Let's produce a nice error message.
+ raise ScannerError("while scanning for the next token", None,
+ "found character %r that cannot start any token" % ch,
+ self.get_mark())
+
+ # Simple keys treatment.
+
+ def next_possible_simple_key(self):
+ # Return the number of the nearest possible simple key. Actually we
+ # don't need to loop through the whole dictionary. We may replace it
+ # with the following code:
+ # if not self.possible_simple_keys:
+ # return None
+ # return self.possible_simple_keys[
+ # min(self.possible_simple_keys.keys())].token_number
+ min_token_number = None
+ for level in self.possible_simple_keys:
+ key = self.possible_simple_keys[level]
+ if min_token_number is None or key.token_number < min_token_number:
+ min_token_number = key.token_number
+ return min_token_number
+
+ def stale_possible_simple_keys(self):
+ # Remove entries that are no longer possible simple keys. According to
+ # the YAML specification, simple keys
+ # - should be limited to a single line,
+ # - should be no longer than 1024 characters.
+ # Disabling this procedure will allow simple keys of any length and
+ # height (may cause problems if indentation is broken though).
+ for level in list(self.possible_simple_keys):
+ key = self.possible_simple_keys[level]
+ if key.line != self.line \
+ or self.index-key.index > 1024:
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not find expected ':'", self.get_mark())
+ del self.possible_simple_keys[level]
+
+ def save_possible_simple_key(self):
+ # The next token may start a simple key. We check if it's possible
+ # and save its position. This function is called for
+ # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
+
+ # Check if a simple key is required at the current position.
+ required = not self.flow_level and self.indent == self.column
+
+ # The next token might be a simple key. Let's save it's number and
+ # position.
+ if self.allow_simple_key:
+ self.remove_possible_simple_key()
+ token_number = self.tokens_taken+len(self.tokens)
+ key = SimpleKey(token_number, required,
+ self.index, self.line, self.column, self.get_mark())
+ self.possible_simple_keys[self.flow_level] = key
+
+ def remove_possible_simple_key(self):
+ # Remove the saved possible key position at the current flow level.
+ if self.flow_level in self.possible_simple_keys:
+ key = self.possible_simple_keys[self.flow_level]
+
+ if key.required:
+ raise ScannerError("while scanning a simple key", key.mark,
+ "could not find expected ':'", self.get_mark())
+
+ del self.possible_simple_keys[self.flow_level]
+
+ # Indentation functions.
+
+ def unwind_indent(self, column):
+
+ ## In flow context, tokens should respect indentation.
+ ## Actually the condition should be `self.indent >= column` according to
+ ## the spec. But this condition will prohibit intuitively correct
+ ## constructions such as
+ ## key : {
+ ## }
+ #if self.flow_level and self.indent > column:
+ # raise ScannerError(None, None,
+ # "invalid indentation or unclosed '[' or '{'",
+ # self.get_mark())
+
+ # In the flow context, indentation is ignored. We make the scanner less
+ # restrictive then specification requires.
+ if self.flow_level:
+ return
+
+ # In block context, we may need to issue the BLOCK-END tokens.
+ while self.indent > column:
+ mark = self.get_mark()
+ self.indent = self.indents.pop()
+ self.tokens.append(BlockEndToken(mark, mark))
+
+ def add_indent(self, column):
+ # Check if we need to increase indentation.
+ if self.indent < column:
+ self.indents.append(self.indent)
+ self.indent = column
+ return True
+ return False
+
+ # Fetchers.
+
+ def fetch_stream_start(self):
+ # We always add STREAM-START as the first token and STREAM-END as the
+ # last token.
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-START.
+ self.tokens.append(StreamStartToken(mark, mark,
+ encoding=self.encoding))
+
+
+ def fetch_stream_end(self):
+
+ # Set the current indentation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+ self.possible_simple_keys = {}
+
+ # Read the token.
+ mark = self.get_mark()
+
+ # Add STREAM-END.
+ self.tokens.append(StreamEndToken(mark, mark))
+
+ # The steam is finished.
+ self.done = True
+
+ def fetch_directive(self):
+
+ # Set the current indentation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Scan and add DIRECTIVE.
+ self.tokens.append(self.scan_directive())
+
+ def fetch_document_start(self):
+ self.fetch_document_indicator(DocumentStartToken)
+
+ def fetch_document_end(self):
+ self.fetch_document_indicator(DocumentEndToken)
+
+ def fetch_document_indicator(self, TokenClass):
+
+ # Set the current indentation to -1.
+ self.unwind_indent(-1)
+
+ # Reset simple keys. Note that there could not be a block collection
+ # after '---'.
+ self.remove_possible_simple_key()
+ self.allow_simple_key = False
+
+ # Add DOCUMENT-START or DOCUMENT-END.
+ start_mark = self.get_mark()
+ self.forward(3)
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_start(self):
+ self.fetch_flow_collection_start(FlowSequenceStartToken)
+
+ def fetch_flow_mapping_start(self):
+ self.fetch_flow_collection_start(FlowMappingStartToken)
+
+ def fetch_flow_collection_start(self, TokenClass):
+
+ # '[' and '{' may start a simple key.
+ self.save_possible_simple_key()
+
+ # Increase the flow level.
+ self.flow_level += 1
+
+ # Simple keys are allowed after '[' and '{'.
+ self.allow_simple_key = True
+
+ # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_sequence_end(self):
+ self.fetch_flow_collection_end(FlowSequenceEndToken)
+
+ def fetch_flow_mapping_end(self):
+ self.fetch_flow_collection_end(FlowMappingEndToken)
+
+ def fetch_flow_collection_end(self, TokenClass):
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Decrease the flow level.
+ self.flow_level -= 1
+
+ # No simple keys after ']' or '}'.
+ self.allow_simple_key = False
+
+ # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(TokenClass(start_mark, end_mark))
+
+ def fetch_flow_entry(self):
+
+ # Simple keys are allowed after ','.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add FLOW-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(FlowEntryToken(start_mark, end_mark))
+
+ def fetch_block_entry(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a new entry?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "sequence entries are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-SEQUENCE-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockSequenceStartToken(mark, mark))
+
+ # It's an error for the block entry to occur in the flow context,
+ # but we let the parser detect this.
+ else:
+ pass
+
+ # Simple keys are allowed after '-'.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add BLOCK-ENTRY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(BlockEntryToken(start_mark, end_mark))
+
+ def fetch_key(self):
+
+ # Block context needs additional checks.
+ if not self.flow_level:
+
+ # Are we allowed to start a key (not necessary a simple)?
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping keys are not allowed here",
+ self.get_mark())
+
+ # We may need to add BLOCK-MAPPING-START.
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after '?' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add KEY.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(KeyToken(start_mark, end_mark))
+
+ def fetch_value(self):
+
+ # Do we determine a simple key?
+ if self.flow_level in self.possible_simple_keys:
+
+ # Add KEY.
+ key = self.possible_simple_keys[self.flow_level]
+ del self.possible_simple_keys[self.flow_level]
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ KeyToken(key.mark, key.mark))
+
+ # If this key starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START.
+ if not self.flow_level:
+ if self.add_indent(key.column):
+ self.tokens.insert(key.token_number-self.tokens_taken,
+ BlockMappingStartToken(key.mark, key.mark))
+
+ # There cannot be two simple keys one after another.
+ self.allow_simple_key = False
+
+ # It must be a part of a complex key.
+ else:
+
+ # Block context needs additional checks.
+ # (Do we really need them? They will be caught by the parser
+ # anyway.)
+ if not self.flow_level:
+
+ # We are allowed to start a complex value if and only if
+ # we can start a simple key.
+ if not self.allow_simple_key:
+ raise ScannerError(None, None,
+ "mapping values are not allowed here",
+ self.get_mark())
+
+ # If this value starts a new block mapping, we need to add
+ # BLOCK-MAPPING-START. It will be detected as an error later by
+ # the parser.
+ if not self.flow_level:
+ if self.add_indent(self.column):
+ mark = self.get_mark()
+ self.tokens.append(BlockMappingStartToken(mark, mark))
+
+ # Simple keys are allowed after ':' in the block context.
+ self.allow_simple_key = not self.flow_level
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Add VALUE.
+ start_mark = self.get_mark()
+ self.forward()
+ end_mark = self.get_mark()
+ self.tokens.append(ValueToken(start_mark, end_mark))
+
+ def fetch_alias(self):
+
+ # ALIAS could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ALIAS.
+ self.allow_simple_key = False
+
+ # Scan and add ALIAS.
+ self.tokens.append(self.scan_anchor(AliasToken))
+
+ def fetch_anchor(self):
+
+ # ANCHOR could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after ANCHOR.
+ self.allow_simple_key = False
+
+ # Scan and add ANCHOR.
+ self.tokens.append(self.scan_anchor(AnchorToken))
+
+ def fetch_tag(self):
+
+ # TAG could start a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after TAG.
+ self.allow_simple_key = False
+
+ # Scan and add TAG.
+ self.tokens.append(self.scan_tag())
+
+ def fetch_literal(self):
+ self.fetch_block_scalar(style='|')
+
+ def fetch_folded(self):
+ self.fetch_block_scalar(style='>')
+
+ def fetch_block_scalar(self, style):
+
+ # A simple key may follow a block scalar.
+ self.allow_simple_key = True
+
+ # Reset possible simple key on the current level.
+ self.remove_possible_simple_key()
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_block_scalar(style))
+
+ def fetch_single(self):
+ self.fetch_flow_scalar(style='\'')
+
+ def fetch_double(self):
+ self.fetch_flow_scalar(style='"')
+
+ def fetch_flow_scalar(self, style):
+
+ # A flow scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after flow scalars.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR.
+ self.tokens.append(self.scan_flow_scalar(style))
+
+ def fetch_plain(self):
+
+ # A plain scalar could be a simple key.
+ self.save_possible_simple_key()
+
+ # No simple keys after plain scalars. But note that `scan_plain` will
+ # change this flag if the scan is finished at the beginning of the
+ # line.
+ self.allow_simple_key = False
+
+ # Scan and add SCALAR. May change `allow_simple_key`.
+ self.tokens.append(self.scan_plain())
+
+ # Checkers.
+
+ def check_directive(self):
+
+ # DIRECTIVE: ^ '%' ...
+ # The '%' indicator is already checked.
+ if self.column == 0:
+ return True
+
+ def check_document_start(self):
+
+ # DOCUMENT-START: ^ '---' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == '---' \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_document_end(self):
+
+ # DOCUMENT-END: ^ '...' (' '|'\n')
+ if self.column == 0:
+ if self.prefix(3) == '...' \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return True
+
+ def check_block_entry(self):
+
+ # BLOCK-ENTRY: '-' (' '|'\n')
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_key(self):
+
+ # KEY(flow context): '?'
+ if self.flow_level:
+ return True
+
+ # KEY(block context): '?' (' '|'\n')
+ else:
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_value(self):
+
+ # VALUE(flow context): ':'
+ if self.flow_level:
+ return True
+
+ # VALUE(block context): ':' (' '|'\n')
+ else:
+ return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+ def check_plain(self):
+
+ # A plain scalar may start with any non-space character except:
+ # '-', '?', ':', ',', '[', ']', '{', '}',
+ # '#', '&', '*', '!', '|', '>', '\'', '\"',
+ # '%', '@', '`'.
+ #
+ # It may also start with
+ # '-', '?', ':'
+ # if it is followed by a non-space character.
+ #
+ # Note that we limit the last rule to the block context (except the
+ # '-' character) because we want the flow context to be space
+ # independent.
+ ch = self.peek()
+ return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \
+ or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029'
+ and (ch == '-' or (not self.flow_level and ch in '?:')))
+
+ # Scanners.
+
+ def scan_to_next_token(self):
+ # We ignore spaces, line breaks and comments.
+ # If we find a line break in the block context, we set the flag
+ # `allow_simple_key` on.
+ # The byte order mark is stripped if it's the first character in the
+ # stream. We do not yet support BOM inside the stream as the
+ # specification requires. Any such mark will be considered as a part
+ # of the document.
+ #
+ # TODO: We need to make tab handling rules more sane. A good rule is
+ # Tabs cannot precede tokens
+ # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
+ # KEY(block), VALUE(block), BLOCK-ENTRY
+ # So the checking code is
+ # if :
+ # self.allow_simple_keys = False
+ # We also need to add the check for `allow_simple_keys == True` to
+ # `unwind_indent` before issuing BLOCK-END.
+ # Scanners for block, flow, and plain scalars need to be modified.
+
+ if self.index == 0 and self.peek() == '\uFEFF':
+ self.forward()
+ found = False
+ while not found:
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ if self.scan_line_break():
+ if not self.flow_level:
+ self.allow_simple_key = True
+ else:
+ found = True
+
+ def scan_directive(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ self.forward()
+ name = self.scan_directive_name(start_mark)
+ value = None
+ if name == 'YAML':
+ value = self.scan_yaml_directive_value(start_mark)
+ end_mark = self.get_mark()
+ elif name == 'TAG':
+ value = self.scan_tag_directive_value(start_mark)
+ end_mark = self.get_mark()
+ else:
+ end_mark = self.get_mark()
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ self.scan_directive_ignored_line(start_mark)
+ return DirectiveToken(name, value, start_mark, end_mark)
+
+ def scan_directive_name(self, start_mark):
+ # See the specification for details.
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ return value
+
+ def scan_yaml_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ major = self.scan_yaml_directive_number(start_mark)
+ if self.peek() != '.':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or '.', but found %r" % self.peek(),
+ self.get_mark())
+ self.forward()
+ minor = self.scan_yaml_directive_number(start_mark)
+ if self.peek() not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit or ' ', but found %r" % self.peek(),
+ self.get_mark())
+ return (major, minor)
+
+ def scan_yaml_directive_number(self, start_mark):
+ # See the specification for details.
+ ch = self.peek()
+ if not ('0' <= ch <= '9'):
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a digit, but found %r" % ch, self.get_mark())
+ length = 0
+ while '0' <= self.peek(length) <= '9':
+ length += 1
+ value = int(self.prefix(length))
+ self.forward(length)
+ return value
+
+ def scan_tag_directive_value(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ handle = self.scan_tag_directive_handle(start_mark)
+ while self.peek() == ' ':
+ self.forward()
+ prefix = self.scan_tag_directive_prefix(start_mark)
+ return (handle, prefix)
+
+ def scan_tag_directive_handle(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_handle('directive', start_mark)
+ ch = self.peek()
+ if ch != ' ':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ return value
+
+ def scan_tag_directive_prefix(self, start_mark):
+ # See the specification for details.
+ value = self.scan_tag_uri('directive', start_mark)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ return value
+
+ def scan_directive_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a directive", start_mark,
+ "expected a comment or a line break, but found %r"
+ % ch, self.get_mark())
+ self.scan_line_break()
+
+ def scan_anchor(self, TokenClass):
+ # The specification does not restrict characters for anchors and
+ # aliases. This may lead to problems, for instance, the document:
+ # [ *alias, value ]
+ # can be interpreted in two ways, as
+ # [ "value" ]
+ # and
+ # [ *alias , "value" ]
+ # Therefore we restrict aliases to numbers and ASCII letters.
+ start_mark = self.get_mark()
+ indicator = self.peek()
+ if indicator == '*':
+ name = 'alias'
+ else:
+ name = 'anchor'
+ self.forward()
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if not length:
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ value = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
+ raise ScannerError("while scanning an %s" % name, start_mark,
+ "expected alphabetic or numeric character, but found %r"
+ % ch, self.get_mark())
+ end_mark = self.get_mark()
+ return TokenClass(value, start_mark, end_mark)
+
+ def scan_tag(self):
+ # See the specification for details.
+ start_mark = self.get_mark()
+ ch = self.peek(1)
+ if ch == '<':
+ handle = None
+ self.forward(2)
+ suffix = self.scan_tag_uri('tag', start_mark)
+ if self.peek() != '>':
+ raise ScannerError("while parsing a tag", start_mark,
+ "expected '>', but found %r" % self.peek(),
+ self.get_mark())
+ self.forward()
+ elif ch in '\0 \t\r\n\x85\u2028\u2029':
+ handle = None
+ suffix = '!'
+ self.forward()
+ else:
+ length = 1
+ use_handle = False
+ while ch not in '\0 \r\n\x85\u2028\u2029':
+ if ch == '!':
+ use_handle = True
+ break
+ length += 1
+ ch = self.peek(length)
+ handle = '!'
+ if use_handle:
+ handle = self.scan_tag_handle('tag', start_mark)
+ else:
+ handle = '!'
+ self.forward()
+ suffix = self.scan_tag_uri('tag', start_mark)
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a tag", start_mark,
+ "expected ' ', but found %r" % ch, self.get_mark())
+ value = (handle, suffix)
+ end_mark = self.get_mark()
+ return TagToken(value, start_mark, end_mark)
+
+ def scan_block_scalar(self, style):
+ # See the specification for details.
+
+ if style == '>':
+ folded = True
+ else:
+ folded = False
+
+ chunks = []
+ start_mark = self.get_mark()
+
+ # Scan the header.
+ self.forward()
+ chomping, increment = self.scan_block_scalar_indicators(start_mark)
+ self.scan_block_scalar_ignored_line(start_mark)
+
+ # Determine the indentation level and go to the first non-empty line.
+ min_indent = self.indent+1
+ if min_indent < 1:
+ min_indent = 1
+ if increment is None:
+ breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
+ indent = max(min_indent, max_indent)
+ else:
+ indent = min_indent+increment-1
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ line_break = ''
+
+ # Scan the inner part of the block scalar.
+ while self.column == indent and self.peek() != '\0':
+ chunks.extend(breaks)
+ leading_non_space = self.peek() not in ' \t'
+ length = 0
+ while self.peek(length) not in '\0\r\n\x85\u2028\u2029':
+ length += 1
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ line_break = self.scan_line_break()
+ breaks, end_mark = self.scan_block_scalar_breaks(indent)
+ if self.column == indent and self.peek() != '\0':
+
+ # Unfortunately, folding rules are ambiguous.
+ #
+ # This is the folding according to the specification:
+
+ if folded and line_break == '\n' \
+ and leading_non_space and self.peek() not in ' \t':
+ if not breaks:
+ chunks.append(' ')
+ else:
+ chunks.append(line_break)
+
+ # This is Clark Evans's interpretation (also in the spec
+ # examples):
+ #
+ #if folded and line_break == '\n':
+ # if not breaks:
+ # if self.peek() not in ' \t':
+ # chunks.append(' ')
+ # else:
+ # chunks.append(line_break)
+ #else:
+ # chunks.append(line_break)
+ else:
+ break
+
+ # Chomp the tail.
+ if chomping is not False:
+ chunks.append(line_break)
+ if chomping is True:
+ chunks.extend(breaks)
+
+ # We are done.
+ return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ def scan_block_scalar_indicators(self, start_mark):
+ # See the specification for details.
+ chomping = None
+ increment = None
+ ch = self.peek()
+ if ch in '+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch in '0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ elif ch in '0123456789':
+ increment = int(ch)
+ if increment == 0:
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected indentation indicator in the range 1-9, but found 0",
+ self.get_mark())
+ self.forward()
+ ch = self.peek()
+ if ch in '+-':
+ if ch == '+':
+ chomping = True
+ else:
+ chomping = False
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0 \r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected chomping or indentation indicators, but found %r"
+ % ch, self.get_mark())
+ return chomping, increment
+
+ def scan_block_scalar_ignored_line(self, start_mark):
+ # See the specification for details.
+ while self.peek() == ' ':
+ self.forward()
+ if self.peek() == '#':
+ while self.peek() not in '\0\r\n\x85\u2028\u2029':
+ self.forward()
+ ch = self.peek()
+ if ch not in '\0\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a block scalar", start_mark,
+ "expected a comment or a line break, but found %r" % ch,
+ self.get_mark())
+ self.scan_line_break()
+
+ def scan_block_scalar_indentation(self):
+ # See the specification for details.
+ chunks = []
+ max_indent = 0
+ end_mark = self.get_mark()
+ while self.peek() in ' \r\n\x85\u2028\u2029':
+ if self.peek() != ' ':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ else:
+ self.forward()
+ if self.column > max_indent:
+ max_indent = self.column
+ return chunks, max_indent, end_mark
+
+ def scan_block_scalar_breaks(self, indent):
+ # See the specification for details.
+ chunks = []
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == ' ':
+ self.forward()
+ while self.peek() in '\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ end_mark = self.get_mark()
+ while self.column < indent and self.peek() == ' ':
+ self.forward()
+ return chunks, end_mark
+
+ def scan_flow_scalar(self, style):
+ # See the specification for details.
+ # Note that we loose indentation rules for quoted scalars. Quoted
+ # scalars don't need to adhere indentation because " and ' clearly
+ # mark the beginning and the end of them. Therefore we are less
+ # restrictive then the specification requires. We only need to check
+ # that document separators are not included in scalars.
+ if style == '"':
+ double = True
+ else:
+ double = False
+ chunks = []
+ start_mark = self.get_mark()
+ quote = self.peek()
+ self.forward()
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ while self.peek() != quote:
+ chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
+ chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+ self.forward()
+ end_mark = self.get_mark()
+ return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+ style)
+
+ ESCAPE_REPLACEMENTS = {
+ '0': '\0',
+ 'a': '\x07',
+ 'b': '\x08',
+ 't': '\x09',
+ '\t': '\x09',
+ 'n': '\x0A',
+ 'v': '\x0B',
+ 'f': '\x0C',
+ 'r': '\x0D',
+ 'e': '\x1B',
+ ' ': '\x20',
+ '\"': '\"',
+ '\\': '\\',
+ '/': '/',
+ 'N': '\x85',
+ '_': '\xA0',
+ 'L': '\u2028',
+ 'P': '\u2029',
+ }
+
+ ESCAPE_CODES = {
+ 'x': 2,
+ 'u': 4,
+ 'U': 8,
+ }
+
+ def scan_flow_scalar_non_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ length = 0
+ while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':
+ length += 1
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ ch = self.peek()
+ if not double and ch == '\'' and self.peek(1) == '\'':
+ chunks.append('\'')
+ self.forward(2)
+ elif (double and ch == '\'') or (not double and ch in '\"\\'):
+ chunks.append(ch)
+ self.forward()
+ elif double and ch == '\\':
+ self.forward()
+ ch = self.peek()
+ if ch in self.ESCAPE_REPLACEMENTS:
+ chunks.append(self.ESCAPE_REPLACEMENTS[ch])
+ self.forward()
+ elif ch in self.ESCAPE_CODES:
+ length = self.ESCAPE_CODES[ch]
+ self.forward()
+ for k in range(length):
+ if self.peek(k) not in '0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "expected escape sequence of %d hexadecimal numbers, but found %r" %
+ (length, self.peek(k)), self.get_mark())
+ code = int(self.prefix(length), 16)
+ chunks.append(chr(code))
+ self.forward(length)
+ elif ch in '\r\n\x85\u2028\u2029':
+ self.scan_line_break()
+ chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
+ else:
+ raise ScannerError("while scanning a double-quoted scalar", start_mark,
+ "found unknown escape character %r" % ch, self.get_mark())
+ else:
+ return chunks
+
+ def scan_flow_scalar_spaces(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ length = 0
+ while self.peek(length) in ' \t':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch == '\0':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected end of stream", self.get_mark())
+ elif ch in '\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ breaks = self.scan_flow_scalar_breaks(double, start_mark)
+ if line_break != '\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(' ')
+ chunks.extend(breaks)
+ else:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_flow_scalar_breaks(self, double, start_mark):
+ # See the specification for details.
+ chunks = []
+ while True:
+ # Instead of checking indentation, we check for document
+ # separators.
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ raise ScannerError("while scanning a quoted scalar", start_mark,
+ "found unexpected document separator", self.get_mark())
+ while self.peek() in ' \t':
+ self.forward()
+ if self.peek() in '\r\n\x85\u2028\u2029':
+ chunks.append(self.scan_line_break())
+ else:
+ return chunks
+
+ def scan_plain(self):
+ # See the specification for details.
+ # We add an additional restriction for the flow context:
+ # plain scalars in the flow context cannot contain ',' or '?'.
+ # We also keep track of the `allow_simple_key` flag here.
+ # Indentation rules are loosed for the flow context.
+ chunks = []
+ start_mark = self.get_mark()
+ end_mark = start_mark
+ indent = self.indent+1
+ # We allow zero indentation for scalars, but then we need to check for
+ # document separators at the beginning of the line.
+ #if indent == 0:
+ # indent = 1
+ spaces = []
+ while True:
+ length = 0
+ if self.peek() == '#':
+ break
+ while True:
+ ch = self.peek(length)
+ if ch in '\0 \t\r\n\x85\u2028\u2029' \
+ or (ch == ':' and
+ self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029'
+ + (u',[]{}' if self.flow_level else u''))\
+ or (self.flow_level and ch in ',?[]{}'):
+ break
+ length += 1
+ if length == 0:
+ break
+ self.allow_simple_key = False
+ chunks.extend(spaces)
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ end_mark = self.get_mark()
+ spaces = self.scan_plain_spaces(indent, start_mark)
+ if not spaces or self.peek() == '#' \
+ or (not self.flow_level and self.column < indent):
+ break
+ return ScalarToken(''.join(chunks), True, start_mark, end_mark)
+
+ def scan_plain_spaces(self, indent, start_mark):
+ # See the specification for details.
+ # The specification is really confusing about tabs in plain scalars.
+ # We just forbid them completely. Do not use tabs in YAML!
+ chunks = []
+ length = 0
+ while self.peek(length) in ' ':
+ length += 1
+ whitespaces = self.prefix(length)
+ self.forward(length)
+ ch = self.peek()
+ if ch in '\r\n\x85\u2028\u2029':
+ line_break = self.scan_line_break()
+ self.allow_simple_key = True
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return
+ breaks = []
+ while self.peek() in ' \r\n\x85\u2028\u2029':
+ if self.peek() == ' ':
+ self.forward()
+ else:
+ breaks.append(self.scan_line_break())
+ prefix = self.prefix(3)
+ if (prefix == '---' or prefix == '...') \
+ and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+ return
+ if line_break != '\n':
+ chunks.append(line_break)
+ elif not breaks:
+ chunks.append(' ')
+ chunks.extend(breaks)
+ elif whitespaces:
+ chunks.append(whitespaces)
+ return chunks
+
+ def scan_tag_handle(self, name, start_mark):
+ # See the specification for details.
+ # For some strange reasons, the specification does not allow '_' in
+ # tag handles. I have allowed it anyway.
+ ch = self.peek()
+ if ch != '!':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch, self.get_mark())
+ length = 1
+ ch = self.peek(length)
+ if ch != ' ':
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-_':
+ length += 1
+ ch = self.peek(length)
+ if ch != '!':
+ self.forward(length)
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected '!', but found %r" % ch, self.get_mark())
+ length += 1
+ value = self.prefix(length)
+ self.forward(length)
+ return value
+
+ def scan_tag_uri(self, name, start_mark):
+ # See the specification for details.
+ # Note: we do not check if URI is well-formed.
+ chunks = []
+ length = 0
+ ch = self.peek(length)
+ while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+ or ch in '-;/?:@&=+$,_.!~*\'()[]%':
+ if ch == '%':
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ chunks.append(self.scan_uri_escapes(name, start_mark))
+ else:
+ length += 1
+ ch = self.peek(length)
+ if length:
+ chunks.append(self.prefix(length))
+ self.forward(length)
+ length = 0
+ if not chunks:
+ raise ScannerError("while parsing a %s" % name, start_mark,
+ "expected URI, but found %r" % ch, self.get_mark())
+ return ''.join(chunks)
+
+ def scan_uri_escapes(self, name, start_mark):
+ # See the specification for details.
+ codes = []
+ mark = self.get_mark()
+ while self.peek() == '%':
+ self.forward()
+ for k in range(2):
+ if self.peek(k) not in '0123456789ABCDEFabcdef':
+ raise ScannerError("while scanning a %s" % name, start_mark,
+ "expected URI escape sequence of 2 hexadecimal numbers, but found %r"
+ % self.peek(k), self.get_mark())
+ codes.append(int(self.prefix(2), 16))
+ self.forward(2)
+ try:
+ value = bytes(codes).decode('utf-8')
+ except UnicodeDecodeError as exc:
+ raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
+ return value
+
+ def scan_line_break(self):
+ # Transforms:
+ # '\r\n' : '\n'
+ # '\r' : '\n'
+ # '\n' : '\n'
+ # '\x85' : '\n'
+ # '\u2028' : '\u2028'
+ # '\u2029 : '\u2029'
+ # default : ''
+ ch = self.peek()
+ if ch in '\r\n\x85':
+ if self.prefix(2) == '\r\n':
+ self.forward(2)
+ else:
+ self.forward()
+ return '\n'
+ elif ch in '\u2028\u2029':
+ self.forward()
+ return ch
+ return ''
diff --git a/phivenv/Lib/site-packages/yaml/serializer.py b/phivenv/Lib/site-packages/yaml/serializer.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe911e67ae7a739abb491fbbc6834b9c37bbda4b
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/serializer.py
@@ -0,0 +1,111 @@
+
+__all__ = ['Serializer', 'SerializerError']
+
+from .error import YAMLError
+from .events import *
+from .nodes import *
+
+class SerializerError(YAMLError):
+ pass
+
+class Serializer:
+
+ ANCHOR_TEMPLATE = 'id%03d'
+
+ def __init__(self, encoding=None,
+ explicit_start=None, explicit_end=None, version=None, tags=None):
+ self.use_encoding = encoding
+ self.use_explicit_start = explicit_start
+ self.use_explicit_end = explicit_end
+ self.use_version = version
+ self.use_tags = tags
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+ self.closed = None
+
+ def open(self):
+ if self.closed is None:
+ self.emit(StreamStartEvent(encoding=self.use_encoding))
+ self.closed = False
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ else:
+ raise SerializerError("serializer is already opened")
+
+ def close(self):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif not self.closed:
+ self.emit(StreamEndEvent())
+ self.closed = True
+
+ #def __del__(self):
+ # self.close()
+
+ def serialize(self, node):
+ if self.closed is None:
+ raise SerializerError("serializer is not opened")
+ elif self.closed:
+ raise SerializerError("serializer is closed")
+ self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
+ version=self.use_version, tags=self.use_tags))
+ self.anchor_node(node)
+ self.serialize_node(node, None, None)
+ self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+ self.serialized_nodes = {}
+ self.anchors = {}
+ self.last_anchor_id = 0
+
+ def anchor_node(self, node):
+ if node in self.anchors:
+ if self.anchors[node] is None:
+ self.anchors[node] = self.generate_anchor(node)
+ else:
+ self.anchors[node] = None
+ if isinstance(node, SequenceNode):
+ for item in node.value:
+ self.anchor_node(item)
+ elif isinstance(node, MappingNode):
+ for key, value in node.value:
+ self.anchor_node(key)
+ self.anchor_node(value)
+
+ def generate_anchor(self, node):
+ self.last_anchor_id += 1
+ return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+ def serialize_node(self, node, parent, index):
+ alias = self.anchors[node]
+ if node in self.serialized_nodes:
+ self.emit(AliasEvent(alias))
+ else:
+ self.serialized_nodes[node] = True
+ self.descend_resolver(parent, index)
+ if isinstance(node, ScalarNode):
+ detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+ default_tag = self.resolve(ScalarNode, node.value, (False, True))
+ implicit = (node.tag == detected_tag), (node.tag == default_tag)
+ self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
+ style=node.style))
+ elif isinstance(node, SequenceNode):
+ implicit = (node.tag
+ == self.resolve(SequenceNode, node.value, True))
+ self.emit(SequenceStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ index = 0
+ for item in node.value:
+ self.serialize_node(item, node, index)
+ index += 1
+ self.emit(SequenceEndEvent())
+ elif isinstance(node, MappingNode):
+ implicit = (node.tag
+ == self.resolve(MappingNode, node.value, True))
+ self.emit(MappingStartEvent(alias, node.tag, implicit,
+ flow_style=node.flow_style))
+ for key, value in node.value:
+ self.serialize_node(key, node, None)
+ self.serialize_node(value, node, key)
+ self.emit(MappingEndEvent())
+ self.ascend_resolver()
+
diff --git a/phivenv/Lib/site-packages/yaml/tokens.py b/phivenv/Lib/site-packages/yaml/tokens.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d0b48a394ac8c019b401516a12f688df361cf90
--- /dev/null
+++ b/phivenv/Lib/site-packages/yaml/tokens.py
@@ -0,0 +1,104 @@
+
+class Token(object):
+ def __init__(self, start_mark, end_mark):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ def __repr__(self):
+ attributes = [key for key in self.__dict__
+ if not key.endswith('_mark')]
+ attributes.sort()
+ arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+ for key in attributes])
+ return '%s(%s)' % (self.__class__.__name__, arguments)
+
+#class BOMToken(Token):
+# id = ''
+
+class DirectiveToken(Token):
+ id = ''
+ def __init__(self, name, value, start_mark, end_mark):
+ self.name = name
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class DocumentStartToken(Token):
+ id = ''
+
+class DocumentEndToken(Token):
+ id = ''
+
+class StreamStartToken(Token):
+ id = ''
+ def __init__(self, start_mark=None, end_mark=None,
+ encoding=None):
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.encoding = encoding
+
+class StreamEndToken(Token):
+ id = ''
+
+class BlockSequenceStartToken(Token):
+ id = ''
+
+class BlockMappingStartToken(Token):
+ id = ''
+
+class BlockEndToken(Token):
+ id = ''
+
+class FlowSequenceStartToken(Token):
+ id = '['
+
+class FlowMappingStartToken(Token):
+ id = '{'
+
+class FlowSequenceEndToken(Token):
+ id = ']'
+
+class FlowMappingEndToken(Token):
+ id = '}'
+
+class KeyToken(Token):
+ id = '?'
+
+class ValueToken(Token):
+ id = ':'
+
+class BlockEntryToken(Token):
+ id = '-'
+
+class FlowEntryToken(Token):
+ id = ','
+
+class AliasToken(Token):
+ id = ''
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class AnchorToken(Token):
+ id = ''
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class TagToken(Token):
+ id = ''
+ def __init__(self, value, start_mark, end_mark):
+ self.value = value
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+
+class ScalarToken(Token):
+ id = ''
+ def __init__(self, value, plain, start_mark, end_mark, style=None):
+ self.value = value
+ self.plain = plain
+ self.start_mark = start_mark
+ self.end_mark = end_mark
+ self.style = style
+
diff --git a/phivenv/Scripts/Activate.ps1 b/phivenv/Scripts/Activate.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..adf86f18891317cc28cf7eb93787cba611b3cee7
--- /dev/null
+++ b/phivenv/Scripts/Activate.ps1
@@ -0,0 +1,437 @@
+<#
+.Synopsis
+Activate a Python virtual environment for the current PowerShell session.
+
+.Description
+Pushes the python executable for a virtual environment to the front of the
+$Env:PATH environment variable and sets the prompt to signify that you are
+in a Python virtual environment. Makes use of the command line switches as
+well as the `pyvenv.cfg` file values present in the virtual environment.
+
+.Parameter VenvDir
+Path to the directory that contains the virtual environment to activate. The
+default value for this is the parent of the directory that the Activate.ps1
+script is located within.
+
+.Parameter Prompt
+The prompt prefix to display when this virtual environment is activated. By
+default, this prompt is the name of the virtual environment folder (VenvDir)
+surrounded by parentheses and followed by a single space (ie. '(.venv) ').
+
+.Example
+Activate.ps1
+Activates the Python virtual environment that contains the Activate.ps1 script.
+
+.Example
+Activate.ps1 -Verbose
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and shows extra information about the activation as it executes.
+
+.Example
+Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
+Activates the Python virtual environment located in the specified location.
+
+.Example
+Activate.ps1 -Prompt "MyPython"
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and prefixes the current prompt with the specified string (surrounded in
+parentheses) while the virtual environment is active.
+
+.Notes
+On Windows, it may be required to enable this Activate.ps1 script by setting the
+execution policy for the user. You can do this by issuing the following PowerShell
+command:
+
+PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+
+For more information on Execution Policies:
+https://go.microsoft.com/fwlink/?LinkID=135170
+
+#>
+Param(
+ [Parameter(Mandatory = $false)]
+ [String]
+ $VenvDir,
+ [Parameter(Mandatory = $false)]
+ [String]
+ $Prompt
+)
+
+<# Function declarations --------------------------------------------------- #>
+
+<#
+.Synopsis
+Remove all shell session elements added by the Activate script, including the
+addition of the virtual environment's Python executable from the beginning of
+the PATH variable.
+
+.Parameter NonDestructive
+If present, do not remove this function from the global namespace for the
+session.
+
+#>
+function global:deactivate ([switch]$NonDestructive) {
+ # Revert to original values
+
+ # The prior prompt:
+ if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
+ Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
+ Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
+ }
+
+ # The prior PYTHONHOME:
+ if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
+ Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
+ Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
+ }
+
+ # The prior PATH:
+ if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
+ Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
+ Remove-Item -Path Env:_OLD_VIRTUAL_PATH
+ }
+
+ # Just remove the VIRTUAL_ENV altogether:
+ if (Test-Path -Path Env:VIRTUAL_ENV) {
+ Remove-Item -Path env:VIRTUAL_ENV
+ }
+
+ # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
+ if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
+ Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
+ }
+
+ # Leave deactivate function in the global namespace if requested:
+ if (-not $NonDestructive) {
+ Remove-Item -Path function:deactivate
+ }
+}
+
+<#
+.Description
+Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
+given folder, and returns them in a map.
+
+For each line in the pyvenv.cfg file, if that line can be parsed into exactly
+two strings separated by `=` (with any amount of whitespace surrounding the =)
+then it is considered a `key = value` line. The left hand string is the key,
+the right hand is the value.
+
+If the value starts with a `'` or a `"` then the first and last character is
+stripped from the value before being captured.
+
+.Parameter ConfigDir
+Path to the directory that contains the `pyvenv.cfg` file.
+#>
+function Get-PyVenvConfig(
+ [String]
+ $ConfigDir
+) {
+ Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
+
+ # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
+ $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
+
+ # An empty map will be returned if no config file is found.
+ $pyvenvConfig = @{ }
+
+ if ($pyvenvConfigPath) {
+
+ Write-Verbose "File exists, parse `key = value` lines"
+ $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
+
+ $pyvenvConfigContent | ForEach-Object {
+ $keyval = $PSItem -split "\s*=\s*", 2
+ if ($keyval[0] -and $keyval[1]) {
+ $val = $keyval[1]
+
+ # Remove extraneous quotations around a string value.
+ if ("'""".Contains($val.Substring(0, 1))) {
+ $val = $val.Substring(1, $val.Length - 2)
+ }
+
+ $pyvenvConfig[$keyval[0]] = $val
+ Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
+ }
+ }
+ }
+ return $pyvenvConfig
+}
+
+
+<# Begin Activate script --------------------------------------------------- #>
+
+# Determine the containing directory of this script
+$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
+$VenvExecDir = Get-Item -Path $VenvExecPath
+
+Write-Verbose "Activation script is located in path: '$VenvExecPath'"
+Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
+Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
+
+# Set values required in priority: CmdLine, ConfigFile, Default
+# First, get the location of the virtual environment, it might not be
+# VenvExecDir if specified on the command line.
+if ($VenvDir) {
+ Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
+}
+else {
+ Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
+ $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
+ Write-Verbose "VenvDir=$VenvDir"
+}
+
+# Next, read the `pyvenv.cfg` file to determine any required value such
+# as `prompt`.
+$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
+
+# Next, set the prompt from the command line, or the config file, or
+# just use the name of the virtual environment folder.
+if ($Prompt) {
+ Write-Verbose "Prompt specified as argument, using '$Prompt'"
+}
+else {
+ Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
+ if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
+ Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
+ $Prompt = $pyvenvCfg['prompt'];
+ }
+ else {
+ Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
+ Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
+ $Prompt = Split-Path -Path $venvDir -Leaf
+ }
+}
+
+Write-Verbose "Prompt = '$Prompt'"
+Write-Verbose "VenvDir='$VenvDir'"
+
+# Deactivate any currently active virtual environment, but leave the
+# deactivate function in place.
+deactivate -nondestructive
+
+# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
+# that there is an activated venv.
+$env:VIRTUAL_ENV = $VenvDir
+
+if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
+
+ Write-Verbose "Setting prompt to '$Prompt'"
+
+ # Set the prompt to include the env name
+ # Make sure _OLD_VIRTUAL_PROMPT is global
+ function global:_OLD_VIRTUAL_PROMPT { "" }
+ Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
+ New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
+
+ function global:prompt {
+ Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
+ _OLD_VIRTUAL_PROMPT
+ }
+}
+
+# Clear PYTHONHOME
+if (Test-Path -Path Env:PYTHONHOME) {
+ Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
+ Remove-Item -Path Env:PYTHONHOME
+}
+
+# Add the venv to the PATH
+Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
+$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"
+
+# SIG # Begin signature block
+# MIIkCQYJKoZIhvcNAQcCoIIj+jCCI/YCAQExDzANBglghkgBZQMEAgEFADB5Bgor
+# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
+# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCD50itNqbOCCDp6
+# 9ZnhKce5X7vV6KL67iKMbGTUZ4nf36CCDi8wggawMIIEmKADAgECAhAIrUCyYNKc
+# TJ9ezam9k67ZMA0GCSqGSIb3DQEBDAUAMGIxCzAJBgNVBAYTAlVTMRUwEwYDVQQK
+# EwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20xITAfBgNV
+# BAMTGERpZ2lDZXJ0IFRydXN0ZWQgUm9vdCBHNDAeFw0yMTA0MjkwMDAwMDBaFw0z
+# NjA0MjgyMzU5NTlaMGkxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwg
+# SW5jLjFBMD8GA1UEAxM4RGlnaUNlcnQgVHJ1c3RlZCBHNCBDb2RlIFNpZ25pbmcg
+# UlNBNDA5NiBTSEEzODQgMjAyMSBDQTEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw
+# ggIKAoICAQDVtC9C0CiteLdd1TlZG7GIQvUzjOs9gZdwxbvEhSYwn6SOaNhc9es0
+# JAfhS0/TeEP0F9ce2vnS1WcaUk8OoVf8iJnBkcyBAz5NcCRks43iCH00fUyAVxJr
+# Q5qZ8sU7H/Lvy0daE6ZMswEgJfMQ04uy+wjwiuCdCcBlp/qYgEk1hz1RGeiQIXhF
+# LqGfLOEYwhrMxe6TSXBCMo/7xuoc82VokaJNTIIRSFJo3hC9FFdd6BgTZcV/sk+F
+# LEikVoQ11vkunKoAFdE3/hoGlMJ8yOobMubKwvSnowMOdKWvObarYBLj6Na59zHh
+# 3K3kGKDYwSNHR7OhD26jq22YBoMbt2pnLdK9RBqSEIGPsDsJ18ebMlrC/2pgVItJ
+# wZPt4bRc4G/rJvmM1bL5OBDm6s6R9b7T+2+TYTRcvJNFKIM2KmYoX7BzzosmJQay
+# g9Rc9hUZTO1i4F4z8ujo7AqnsAMrkbI2eb73rQgedaZlzLvjSFDzd5Ea/ttQokbI
+# YViY9XwCFjyDKK05huzUtw1T0PhH5nUwjewwk3YUpltLXXRhTT8SkXbev1jLchAp
+# QfDVxW0mdmgRQRNYmtwmKwH0iU1Z23jPgUo+QEdfyYFQc4UQIyFZYIpkVMHMIRro
+# OBl8ZhzNeDhFMJlP/2NPTLuqDQhTQXxYPUez+rbsjDIJAsxsPAxWEQIDAQABo4IB
+# WTCCAVUwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUaDfg67Y7+F8Rhvv+
+# YXsIiGX0TkIwHwYDVR0jBBgwFoAU7NfjgtJxXWRM3y5nP+e6mK4cD08wDgYDVR0P
+# AQH/BAQDAgGGMBMGA1UdJQQMMAoGCCsGAQUFBwMDMHcGCCsGAQUFBwEBBGswaTAk
+# BggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQuY29tMEEGCCsGAQUFBzAC
+# hjVodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGlnaUNlcnRUcnVzdGVkUm9v
+# dEc0LmNydDBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vY3JsMy5kaWdpY2VydC5j
+# b20vRGlnaUNlcnRUcnVzdGVkUm9vdEc0LmNybDAcBgNVHSAEFTATMAcGBWeBDAED
+# MAgGBmeBDAEEATANBgkqhkiG9w0BAQwFAAOCAgEAOiNEPY0Idu6PvDqZ01bgAhql
+# +Eg08yy25nRm95RysQDKr2wwJxMSnpBEn0v9nqN8JtU3vDpdSG2V1T9J9Ce7FoFF
+# UP2cvbaF4HZ+N3HLIvdaqpDP9ZNq4+sg0dVQeYiaiorBtr2hSBh+3NiAGhEZGM1h
+# mYFW9snjdufE5BtfQ/g+lP92OT2e1JnPSt0o618moZVYSNUa/tcnP/2Q0XaG3Ryw
+# YFzzDaju4ImhvTnhOE7abrs2nfvlIVNaw8rpavGiPttDuDPITzgUkpn13c5Ubdld
+# AhQfQDN8A+KVssIhdXNSy0bYxDQcoqVLjc1vdjcshT8azibpGL6QB7BDf5WIIIJw
+# 8MzK7/0pNVwfiThV9zeKiwmhywvpMRr/LhlcOXHhvpynCgbWJme3kuZOX956rEnP
+# LqR0kq3bPKSchh/jwVYbKyP/j7XqiHtwa+aguv06P0WmxOgWkVKLQcBIhEuWTatE
+# QOON8BUozu3xGFYHKi8QxAwIZDwzj64ojDzLj4gLDb879M4ee47vtevLt/B3E+bn
+# KD+sEq6lLyJsQfmCXBVmzGwOysWGw/YmMwwHS6DTBwJqakAwSEs0qFEgu60bhQji
+# WQ1tygVQK+pKHJ6l/aCnHwZ05/LWUpD9r4VIIflXO7ScA+2GRfS0YW6/aOImYIbq
+# yK+p/pQd52MbOoZWeE4wggd3MIIFX6ADAgECAhAHHxQbizANJfMU6yMM0NHdMA0G
+# CSqGSIb3DQEBCwUAMGkxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwg
+# SW5jLjFBMD8GA1UEAxM4RGlnaUNlcnQgVHJ1c3RlZCBHNCBDb2RlIFNpZ25pbmcg
+# UlNBNDA5NiBTSEEzODQgMjAyMSBDQTEwHhcNMjIwMTE3MDAwMDAwWhcNMjUwMTE1
+# MjM1OTU5WjB8MQswCQYDVQQGEwJVUzEPMA0GA1UECBMGT3JlZ29uMRIwEAYDVQQH
+# EwlCZWF2ZXJ0b24xIzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9u
+# MSMwIQYDVQQDExpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjCCAiIwDQYJKoZI
+# hvcNAQEBBQADggIPADCCAgoCggIBAKgc0BTT+iKbtK6f2mr9pNMUTcAJxKdsuOiS
+# YgDFfwhjQy89koM7uP+QV/gwx8MzEt3c9tLJvDccVWQ8H7mVsk/K+X+IufBLCgUi
+# 0GGAZUegEAeRlSXxxhYScr818ma8EvGIZdiSOhqjYc4KnfgfIS4RLtZSrDFG2tN1
+# 6yS8skFa3IHyvWdbD9PvZ4iYNAS4pjYDRjT/9uzPZ4Pan+53xZIcDgjiTwOh8VGu
+# ppxcia6a7xCyKoOAGjvCyQsj5223v1/Ig7Dp9mGI+nh1E3IwmyTIIuVHyK6Lqu35
+# 2diDY+iCMpk9ZanmSjmB+GMVs+H/gOiofjjtf6oz0ki3rb7sQ8fTnonIL9dyGTJ0
+# ZFYKeb6BLA66d2GALwxZhLe5WH4Np9HcyXHACkppsE6ynYjTOd7+jN1PRJahN1oE
+# RzTzEiV6nCO1M3U1HbPTGyq52IMFSBM2/07WTJSbOeXjvYR7aUxK9/ZkJiacl2iZ
+# I7IWe7JKhHohqKuceQNyOzxTakLcRkzynvIrk33R9YVqtB4L6wtFxhUjvDnQg16x
+# ot2KVPdfyPAWd81wtZADmrUtsZ9qG79x1hBdyOl4vUtVPECuyhCxaw+faVjumapP
+# Unwo8ygflJJ74J+BYxf6UuD7m8yzsfXWkdv52DjL74TxzuFTLHPyARWCSCAbzn3Z
+# Ily+qIqDAgMBAAGjggIGMIICAjAfBgNVHSMEGDAWgBRoN+Drtjv4XxGG+/5hewiI
+# ZfROQjAdBgNVHQ4EFgQUt/1Teh2XDuUj2WW3siYWJgkZHA8wDgYDVR0PAQH/BAQD
+# AgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMIG1BgNVHR8Ega0wgaowU6BRoE+GTWh0
+# dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRHNENvZGVTaWdu
+# aW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3JsMFOgUaBPhk1odHRwOi8vY3JsNC5k
+# aWdpY2VydC5jb20vRGlnaUNlcnRUcnVzdGVkRzRDb2RlU2lnbmluZ1JTQTQwOTZT
+# SEEzODQyMDIxQ0ExLmNybDA+BgNVHSAENzA1MDMGBmeBDAEEATApMCcGCCsGAQUF
+# BwIBFhtodHRwOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwgZQGCCsGAQUFBwEBBIGH
+# MIGEMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wXAYIKwYB
+# BQUHMAKGUGh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0
+# ZWRHNENvZGVTaWduaW5nUlNBNDA5NlNIQTM4NDIwMjFDQTEuY3J0MAwGA1UdEwEB
+# /wQCMAAwDQYJKoZIhvcNAQELBQADggIBABxv4AeV/5ltkELHSC63fXAFYS5tadcW
+# TiNc2rskrNLrfH1Ns0vgSZFoQxYBFKI159E8oQQ1SKbTEubZ/B9kmHPhprHya08+
+# VVzxC88pOEvz68nA82oEM09584aILqYmj8Pj7h/kmZNzuEL7WiwFa/U1hX+XiWfL
+# IJQsAHBla0i7QRF2de8/VSF0XXFa2kBQ6aiTsiLyKPNbaNtbcucaUdn6vVUS5izW
+# OXM95BSkFSKdE45Oq3FForNJXjBvSCpwcP36WklaHL+aHu1upIhCTUkzTHMh8b86
+# WmjRUqbrnvdyR2ydI5l1OqcMBjkpPpIV6wcc+KY/RH2xvVuuoHjlUjwq2bHiNoX+
+# W1scCpnA8YTs2d50jDHUgwUo+ciwpffH0Riq132NFmrH3r67VaN3TuBxjI8SIZM5
+# 8WEDkbeoriDk3hxU8ZWV7b8AW6oyVBGfM06UgkfMb58h+tJPrFx8VI/WLq1dTqMf
+# ZOm5cuclMnUHs2uqrRNtnV8UfidPBL4ZHkTcClQbCoz0UbLhkiDvIS00Dn+BBcxw
+# /TKqVL4Oaz3bkMSsM46LciTeucHY9ExRVt3zy7i149sd+F4QozPqn7FrSVHXmem3
+# r7bjyHTxOgqxRCVa18Vtx7P/8bYSBeS+WHCKcliFCecspusCDSlnRUjZwyPdP0VH
+# xaZg2unjHY3rMYIVMDCCFSwCAQEwfTBpMQswCQYDVQQGEwJVUzEXMBUGA1UEChMO
+# RGlnaUNlcnQsIEluYy4xQTA/BgNVBAMTOERpZ2lDZXJ0IFRydXN0ZWQgRzQgQ29k
+# ZSBTaWduaW5nIFJTQTQwOTYgU0hBMzg0IDIwMjEgQ0ExAhAHHxQbizANJfMU6yMM
+# 0NHdMA0GCWCGSAFlAwQCAQUAoIHOMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEE
+# MBwGCisGAQQBgjcCAQsxDjAMBgorBgEEAYI3AgEVMC8GCSqGSIb3DQEJBDEiBCBW
+# STMEpY5oTqEtnhE8rXHTfUpY7MZMEfDbBKbS1J0cLTBiBgorBgEEAYI3AgEMMVQw
+# UqBQgE4AQgB1AGkAbAB0ADoAIABSAGUAbABlAGEAcwBlAF8AbQBhAGkAbgBfAHYA
+# MwAuADkALgAxADMAXwAyADAAMgAyADAANQAxADcALgAwADIwDQYJKoZIhvcNAQEB
+# BQAEggIAZ20aZkGkTuZACtVG89TVf+3Nw7cpJO4pkpM71AFEX7/TZ4NjeFMHCrEr
+# g8kNhN1UNcf0FGKplenCcykaIYUTlYgT3VCOeRGINOmySIotyEfYw4HapwNhj8/G
+# CK/XOKJvIZ96yHHkpaiCOx3JDVtX5UjkChX1IJNECy0RyzALQA9U/EQl7v8oNEs6
+# mCHRfPflHgkPYUPYUJXAGk3tD0ZgeeomUR0zm6Zmvqar2RJaZADtCF5OxOcFbdui
+# 9yrlbSlkxv6gSW60W372FRCIoy1BML7Okjv7QJnhJkPIAs4sb0ZCV3rB6ZjKBXx3
+# WzeQ5yjpJ3O1ZsZKPP4XVayPJCv2PSRd8dlDgmJtMbN2gvjsLtOkDKN03jnEsiiX
+# 9IeYu7AaDdDeMDW8U5A0HYStrVX7OYpLqojtWOdeIz3/thj9ncWDio4KqEZTud37
+# kfhDQPks9zWHoeI8MnjWMgBCEMskNFQnEhgGNlxWRW8bzypODhlrUGB271/6tFCE
+# UW24irgtfCvNVmQh5E7V+GNplFujgCa7O4wQvYBb1i2OnQ+igJkfWx2N+wmb8QYE
+# mN4RM/bk/SOc1b3MnU/ztqscOPPllmQsqMe/8LW77Ww2fy3i88RlcjweWiTs+3AK
+# akFenQ/OzGPTrG8+mMKEjRKuxKyL2uZRKUoZ2zVlmjEddgnIVCWhghGzMIIRrwYK
+# KwYBBAGCNwMDATGCEZ8wghGbBgkqhkiG9w0BBwKgghGMMIIRiAIBAzEPMA0GCWCG
+# SAFlAwQCAQUAMHgGCyqGSIb3DQEJEAEEoGkEZzBlAgEBBglghkgBhv1sBwEwMTAN
+# BglghkgBZQMEAgEFAAQgnT/1Hedipjs4jpirOCgW93RhvoYu0jJ5WKO7Y0LYfrYC
+# EQDwVmhL49/+ciLQ3plX3i8KGA8yMDIyMDUxNzE2NDQyNVqggg18MIIGxjCCBK6g
+# AwIBAgIQCnpKiJ7JmUKQBmM4TYaXnTANBgkqhkiG9w0BAQsFADBjMQswCQYDVQQG
+# EwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xOzA5BgNVBAMTMkRpZ2lDZXJ0
+# IFRydXN0ZWQgRzQgUlNBNDA5NiBTSEEyNTYgVGltZVN0YW1waW5nIENBMB4XDTIy
+# MDMyOTAwMDAwMFoXDTMzMDMxNDIzNTk1OVowTDELMAkGA1UEBhMCVVMxFzAVBgNV
+# BAoTDkRpZ2lDZXJ0LCBJbmMuMSQwIgYDVQQDExtEaWdpQ2VydCBUaW1lc3RhbXAg
+# MjAyMiAtIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC5KpYjply8
+# X9ZJ8BWCGPQz7sxcbOPgJS7SMeQ8QK77q8TjeF1+XDbq9SWNQ6OB6zhj+TyIad48
+# 0jBRDTEHukZu6aNLSOiJQX8Nstb5hPGYPgu/CoQScWyhYiYB087DbP2sO37cKhyp
+# vTDGFtjavOuy8YPRn80JxblBakVCI0Fa+GDTZSw+fl69lqfw/LH09CjPQnkfO8eT
+# B2ho5UQ0Ul8PUN7UWSxEdMAyRxlb4pguj9DKP//GZ888k5VOhOl2GJiZERTFKwyg
+# M9tNJIXogpThLwPuf4UCyYbh1RgUtwRF8+A4vaK9enGY7BXn/S7s0psAiqwdjTuA
+# aP7QWZgmzuDtrn8oLsKe4AtLyAjRMruD+iM82f/SjLv3QyPf58NaBWJ+cCzlK7I9
+# Y+rIroEga0OJyH5fsBrdGb2fdEEKr7mOCdN0oS+wVHbBkE+U7IZh/9sRL5IDMM4w
+# t4sPXUSzQx0jUM2R1y+d+/zNscGnxA7E70A+GToC1DGpaaBJ+XXhm+ho5GoMj+vk
+# sSF7hmdYfn8f6CvkFLIW1oGhytowkGvub3XAsDYmsgg7/72+f2wTGN/GbaR5Sa2L
+# f2GHBWj31HDjQpXonrubS7LitkE956+nGijJrWGwoEEYGU7tR5thle0+C2Fa6j56
+# mJJRzT/JROeAiylCcvd5st2E6ifu/n16awIDAQABo4IBizCCAYcwDgYDVR0PAQH/
+# BAQDAgeAMAwGA1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwIAYD
+# VR0gBBkwFzAIBgZngQwBBAIwCwYJYIZIAYb9bAcBMB8GA1UdIwQYMBaAFLoW2W1N
+# hS9zKXaaL3WMaiCPnshvMB0GA1UdDgQWBBSNZLeJIf5WWESEYafqbxw2j92vDTBa
+# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNl
+# cnRUcnVzdGVkRzRSU0E0MDk2U0hBMjU2VGltZVN0YW1waW5nQ0EuY3JsMIGQBggr
+# BgEFBQcBAQSBgzCBgDAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu
+# Y29tMFgGCCsGAQUFBzAChkxodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln
+# aUNlcnRUcnVzdGVkRzRSU0E0MDk2U0hBMjU2VGltZVN0YW1waW5nQ0EuY3J0MA0G
+# CSqGSIb3DQEBCwUAA4ICAQANLSN0ptH1+OpLmT8B5PYM5K8WndmzjJeCKZxDbwEt
+# qzi1cBG/hBmLP13lhk++kzreKjlaOU7YhFmlvBuYquhs79FIaRk4W8+JOR1wcNlO
+# 3yMibNXf9lnLocLqTHbKodyhK5a4m1WpGmt90fUCCU+C1qVziMSYgN/uSZW3s8zF
+# p+4O4e8eOIqf7xHJMUpYtt84fMv6XPfkU79uCnx+196Y1SlliQ+inMBl9AEiZcfq
+# XnSmWzWSUHz0F6aHZE8+RokWYyBry/J70DXjSnBIqbbnHWC9BCIVJXAGcqlEO2lH
+# EdPu6cegPk8QuTA25POqaQmoi35komWUEftuMvH1uzitzcCTEdUyeEpLNypM81zc
+# toXAu3AwVXjWmP5UbX9xqUgaeN1Gdy4besAzivhKKIwSqHPPLfnTI/KeGeANlCig
+# 69saUaCVgo4oa6TOnXbeqXOqSGpZQ65f6vgPBkKd3wZolv4qoHRbY2beayy4eKpN
+# cG3wLPEHFX41tOa1DKKZpdcVazUOhdbgLMzgDCS4fFILHpl878jIxYxYaa+rPeHP
+# zH0VrhS/inHfypex2EfqHIXgRU4SHBQpWMxv03/LvsEOSm8gnK7ZczJZCOctkqEa
+# Ef4ymKZdK5fgi9OczG21Da5HYzhHF1tvE9pqEG4fSbdEW7QICodaWQR2EaGndwIT
+# HDCCBq4wggSWoAMCAQICEAc2N7ckVHzYR6z9KGYqXlswDQYJKoZIhvcNAQELBQAw
+# YjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQ
+# d3d3LmRpZ2ljZXJ0LmNvbTEhMB8GA1UEAxMYRGlnaUNlcnQgVHJ1c3RlZCBSb290
+# IEc0MB4XDTIyMDMyMzAwMDAwMFoXDTM3MDMyMjIzNTk1OVowYzELMAkGA1UEBhMC
+# VVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJbmMuMTswOQYDVQQDEzJEaWdpQ2VydCBU
+# cnVzdGVkIEc0IFJTQTQwOTYgU0hBMjU2IFRpbWVTdGFtcGluZyBDQTCCAiIwDQYJ
+# KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMaGNQZJs8E9cklRVcclA8TykTepl1Gh
+# 1tKD0Z5Mom2gsMyD+Vr2EaFEFUJfpIjzaPp985yJC3+dH54PMx9QEwsmc5Zt+Feo
+# An39Q7SE2hHxc7Gz7iuAhIoiGN/r2j3EF3+rGSs+QtxnjupRPfDWVtTnKC3r07G1
+# decfBmWNlCnT2exp39mQh0YAe9tEQYncfGpXevA3eZ9drMvohGS0UvJ2R/dhgxnd
+# X7RUCyFobjchu0CsX7LeSn3O9TkSZ+8OpWNs5KbFHc02DVzV5huowWR0QKfAcsW6
+# Th+xtVhNef7Xj3OTrCw54qVI1vCwMROpVymWJy71h6aPTnYVVSZwmCZ/oBpHIEPj
+# Q2OAe3VuJyWQmDo4EbP29p7mO1vsgd4iFNmCKseSv6De4z6ic/rnH1pslPJSlREr
+# WHRAKKtzQ87fSqEcazjFKfPKqpZzQmiftkaznTqj1QPgv/CiPMpC3BhIfxQ0z9JM
+# q++bPf4OuGQq+nUoJEHtQr8FnGZJUlD0UfM2SU2LINIsVzV5K6jzRWC8I41Y99xh
+# 3pP+OcD5sjClTNfpmEpYPtMDiP6zj9NeS3YSUZPJjAw7W4oiqMEmCPkUEBIDfV8j
+# u2TjY+Cm4T72wnSyPx4JduyrXUZ14mCjWAkBKAAOhFTuzuldyF4wEr1GnrXTdrnS
+# DmuZDNIztM2xAgMBAAGjggFdMIIBWTASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1Ud
+# DgQWBBS6FtltTYUvcyl2mi91jGogj57IbzAfBgNVHSMEGDAWgBTs1+OC0nFdZEzf
+# Lmc/57qYrhwPTzAOBgNVHQ8BAf8EBAMCAYYwEwYDVR0lBAwwCgYIKwYBBQUHAwgw
+# dwYIKwYBBQUHAQEEazBpMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy
+# dC5jb20wQQYIKwYBBQUHMAKGNWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E
+# aWdpQ2VydFRydXN0ZWRSb290RzQuY3J0MEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6
+# Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRydXN0ZWRSb290RzQuY3JsMCAG
+# A1UdIAQZMBcwCAYGZ4EMAQQCMAsGCWCGSAGG/WwHATANBgkqhkiG9w0BAQsFAAOC
+# AgEAfVmOwJO2b5ipRCIBfmbW2CFC4bAYLhBNE88wU86/GPvHUF3iSyn7cIoNqilp
+# /GnBzx0H6T5gyNgL5Vxb122H+oQgJTQxZ822EpZvxFBMYh0MCIKoFr2pVs8Vc40B
+# IiXOlWk/R3f7cnQU1/+rT4osequFzUNf7WC2qk+RZp4snuCKrOX9jLxkJodskr2d
+# fNBwCnzvqLx1T7pa96kQsl3p/yhUifDVinF2ZdrM8HKjI/rAJ4JErpknG6skHibB
+# t94q6/aesXmZgaNWhqsKRcnfxI2g55j7+6adcq/Ex8HBanHZxhOACcS2n82HhyS7
+# T6NJuXdmkfFynOlLAlKnN36TU6w7HQhJD5TNOXrd/yVjmScsPT9rp/Fmw0HNT7ZA
+# myEhQNC3EyTN3B14OuSereU0cZLXJmvkOHOrpgFPvT87eK1MrfvElXvtCl8zOYdB
+# eHo46Zzh3SP9HSjTx/no8Zhf+yvYfvJGnXUsHicsJttvFXseGYs2uJPU5vIXmVnK
+# cPA3v5gA3yAWTyf7YGcWoWa63VXAOimGsJigK+2VQbc61RWYMbRiCQ8KvYHZE/6/
+# pNHzV9m8BPqC3jLfBInwAM1dwvnQI38AC+R2AibZ8GV2QqYphwlHK+Z/GqSFD/yY
+# lvZVVCsfgPrA8g4r5db7qS9EFUrnEw4d2zc4GqEr9u3WfPwxggN2MIIDcgIBATB3
+# MGMxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwgSW5jLjE7MDkGA1UE
+# AxMyRGlnaUNlcnQgVHJ1c3RlZCBHNCBSU0E0MDk2IFNIQTI1NiBUaW1lU3RhbXBp
+# bmcgQ0ECEAp6SoieyZlCkAZjOE2Gl50wDQYJYIZIAWUDBAIBBQCggdEwGgYJKoZI
+# hvcNAQkDMQ0GCyqGSIb3DQEJEAEEMBwGCSqGSIb3DQEJBTEPFw0yMjA1MTcxNjQ0
+# MjVaMCsGCyqGSIb3DQEJEAIMMRwwGjAYMBYEFIUI84ZRXLPTB322tLfAfxtKXkHe
+# MC8GCSqGSIb3DQEJBDEiBCD6nFFQEG6P9dJgqwYWRs4qcIuEc8WzVzP/ANrLjNw9
+# 9DA3BgsqhkiG9w0BCRACLzEoMCYwJDAiBCCdppAVw0nGwYl4Rbo1gq1wyI+kKTvb
+# ar6cK9JTknnmOzANBgkqhkiG9w0BAQEFAASCAgAw6wjOhvvgNacByXWLXEcHuoRB
+# hCB6I/ZNStOapAzlidSg0ccPO4QsS5L78/ff+DCa4ZFu//T2wfORd9wCt9NpL0XT
+# 7/zVF3zw16aNIn1W9gbJkjPfHGLImV30M/PzMnzlj3m9ybK0W/vvu8SmNpMrsSKI
+# sR1/nSzaA+Y0FmOOy2jtf7MtZeVh8o4ZkgBVPgVLpRSr4SJXzadMjQjshWII2ujM
+# v2YPK7qczQmXaonfN+rAkPxLS+3MVDgoUKAGvQBpm686eDkImZ0qTG+Qi6+Z+2id
+# QXv8h4V4L7Ln37RNsm4j5carTNxTx6b0SZPreswMEFfUwtHeaSWvUIKRzyY0t3O/
+# NT5G8RSxtJLqTMVFWcmlbgaXo7/PDsA7hs1XQJE3UJnfRNrcznnHgYJPuW737A9Q
+# v4sFjmov5/F4qUbYIh5yE8ec4IhOuR0rTEgDJLcU67KdrLjcePMlIlbPaArw1vn3
+# On862t4jYz5aj3YaaHYzXB6VxiMp9JN5B3e4gmix/8TgN9kNmvbVwODfK5hfaPcY
+# QXkTBySKKoRz+eyz67IBXPCHDYXtiXdaqvhWPFn+YbFH9jlEqPP2ATzqEz/ibgDZ
+# 0YyvPy8vnItbUwPuGTVjwU0VmF0m5pSHm/pPALd1xKiMF+eZq580lbrIoUeyZ5mP
+# 0XyrSEj88tBJMxcKZA==
+# SIG # End signature block
diff --git a/phivenv/Scripts/activate b/phivenv/Scripts/activate
new file mode 100644
index 0000000000000000000000000000000000000000..aaee3311bfdf068122125be9d8c0a068558c458d
--- /dev/null
+++ b/phivenv/Scripts/activate
@@ -0,0 +1,66 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+ # reset old environment variables
+ if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
+ PATH="${_OLD_VIRTUAL_PATH:-}"
+ export PATH
+ unset _OLD_VIRTUAL_PATH
+ fi
+ if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
+ PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
+ export PYTHONHOME
+ unset _OLD_VIRTUAL_PYTHONHOME
+ fi
+
+ # This should detect bash and zsh, which have a hash command that must
+ # be called to get it to forget past commands. Without forgetting
+ # past commands the $PATH changes we made may not be respected
+ if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+ hash -r 2> /dev/null
+ fi
+
+ if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
+ PS1="${_OLD_VIRTUAL_PS1:-}"
+ export PS1
+ unset _OLD_VIRTUAL_PS1
+ fi
+
+ unset VIRTUAL_ENV
+ if [ ! "${1:-}" = "nondestructive" ] ; then
+ # Self destruct!
+ unset -f deactivate
+ fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="D:\phi2_tuning\phivenv"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/Scripts:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "${PYTHONHOME:-}" ] ; then
+ _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
+ unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
+ _OLD_VIRTUAL_PS1="${PS1:-}"
+ PS1="(phivenv) ${PS1:-}"
+ export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands. Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+ hash -r 2> /dev/null
+fi
diff --git a/phivenv/Scripts/activate.bat b/phivenv/Scripts/activate.bat
new file mode 100644
index 0000000000000000000000000000000000000000..90d7e11cc75531d16a360fa0ef0e482bf3da1dcc
--- /dev/null
+++ b/phivenv/Scripts/activate.bat
@@ -0,0 +1,33 @@
+@echo off
+
+rem This file is UTF-8 encoded, so we need to update the current code page while executing it
+for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
+ set _OLD_CODEPAGE=%%a
+)
+if defined _OLD_CODEPAGE (
+ "%SystemRoot%\System32\chcp.com" 65001 > nul
+)
+
+set VIRTUAL_ENV=D:\phi2_tuning\phivenv
+
+if not defined PROMPT set PROMPT=$P$G
+
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
+
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=(phivenv) %PROMPT%
+
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
+
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH%
+
+set PATH=%VIRTUAL_ENV%\Scripts;%PATH%
+
+:END
+if defined _OLD_CODEPAGE (
+ "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
+ set _OLD_CODEPAGE=
+)
diff --git a/phivenv/Scripts/deactivate.bat b/phivenv/Scripts/deactivate.bat
new file mode 100644
index 0000000000000000000000000000000000000000..313c0791173682d387fd0657ef4a4c1ff63e7d70
--- /dev/null
+++ b/phivenv/Scripts/deactivate.bat
@@ -0,0 +1,21 @@
+@echo off
+
+if defined _OLD_VIRTUAL_PROMPT (
+ set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
+)
+set _OLD_VIRTUAL_PROMPT=
+
+if defined _OLD_VIRTUAL_PYTHONHOME (
+ set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
+ set _OLD_VIRTUAL_PYTHONHOME=
+)
+
+if defined _OLD_VIRTUAL_PATH (
+ set "PATH=%_OLD_VIRTUAL_PATH%"
+)
+
+set _OLD_VIRTUAL_PATH=
+
+set VIRTUAL_ENV=
+
+:END
diff --git a/phivenv/Scripts/f2py.exe b/phivenv/Scripts/f2py.exe
new file mode 100644
index 0000000000000000000000000000000000000000..268808b385170af03afe327027c536eba37991b0
--- /dev/null
+++ b/phivenv/Scripts/f2py.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed2a8eec2e6452a3ab62cb5b929886a5fe485402d750ec3989abce89979480b4
+size 106347
diff --git a/phivenv/Scripts/hf.exe b/phivenv/Scripts/hf.exe
new file mode 100644
index 0000000000000000000000000000000000000000..8d8f8e4ad99e5d13e1299b6892239ccf07516e55
--- /dev/null
+++ b/phivenv/Scripts/hf.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:001d6d44e4ef09b2184be4d97e6c383920ba737cfbc7fa14c55b37d103c68f43
+size 106352
diff --git a/phivenv/Scripts/huggingface-cli.exe b/phivenv/Scripts/huggingface-cli.exe
new file mode 100644
index 0000000000000000000000000000000000000000..96799717c2879cdd20561f88be2e6b738713891d
--- /dev/null
+++ b/phivenv/Scripts/huggingface-cli.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcaeb48f435d38354eabe6d70a6ab9e61126e56d3829662cc54f659662d66358
+size 106370
diff --git a/phivenv/Scripts/isympy.exe b/phivenv/Scripts/isympy.exe
new file mode 100644
index 0000000000000000000000000000000000000000..2f49da4cd28bc9f25c4461f577b495aaedbe96bd
--- /dev/null
+++ b/phivenv/Scripts/isympy.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb04ea2702ef9ebe0e0e07897d13887f0a116d758d67cf60e8021a2d3e3c9705
+size 106336
diff --git a/phivenv/Scripts/normalizer.exe b/phivenv/Scripts/normalizer.exe
new file mode 100644
index 0000000000000000000000000000000000000000..fa767bb5290c00f94922e394e21c0cb12949d1e4
--- /dev/null
+++ b/phivenv/Scripts/normalizer.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22f7cbd712302dc37c01d8dcceba1d35a5cb687054111581821ed86dfa89614c
+size 106364
diff --git a/phivenv/Scripts/numpy-config.exe b/phivenv/Scripts/numpy-config.exe
new file mode 100644
index 0000000000000000000000000000000000000000..83a0b4995e555d97a78a68f5f314befe9d452785
--- /dev/null
+++ b/phivenv/Scripts/numpy-config.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:787977bcefce9885d23f33c9aca6670071654646e6b075bf1b4a228d931b6962
+size 106347
diff --git a/phivenv/Scripts/pip.exe b/phivenv/Scripts/pip.exe
new file mode 100644
index 0000000000000000000000000000000000000000..b62ae4fefc9338750d82fdd37915108c01ab5bc7
--- /dev/null
+++ b/phivenv/Scripts/pip.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8cbefa2e058f696cdf5433398073d767b4558db17f98919375da9ca2fa09650
+size 106352
diff --git a/phivenv/Scripts/pip3.9.exe b/phivenv/Scripts/pip3.9.exe
new file mode 100644
index 0000000000000000000000000000000000000000..b62ae4fefc9338750d82fdd37915108c01ab5bc7
--- /dev/null
+++ b/phivenv/Scripts/pip3.9.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8cbefa2e058f696cdf5433398073d767b4558db17f98919375da9ca2fa09650
+size 106352
diff --git a/phivenv/Scripts/pip3.exe b/phivenv/Scripts/pip3.exe
new file mode 100644
index 0000000000000000000000000000000000000000..b62ae4fefc9338750d82fdd37915108c01ab5bc7
--- /dev/null
+++ b/phivenv/Scripts/pip3.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8cbefa2e058f696cdf5433398073d767b4558db17f98919375da9ca2fa09650
+size 106352
diff --git a/phivenv/Scripts/python.exe b/phivenv/Scripts/python.exe
new file mode 100644
index 0000000000000000000000000000000000000000..6482d44f0d9e1bb2b066b3da4127542dc8cb56e0
--- /dev/null
+++ b/phivenv/Scripts/python.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3515e307cf21fe8f37112b3a6e79c3c3b50aa202cc000df4bbf3343f083a2f70
+size 599032
diff --git a/phivenv/Scripts/pythonw.exe b/phivenv/Scripts/pythonw.exe
new file mode 100644
index 0000000000000000000000000000000000000000..d3bcb850a1e8a08bcf27a158a9fb97cf79ac3c5c
--- /dev/null
+++ b/phivenv/Scripts/pythonw.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:590a0a5cf16617bdfdaee71d8cbb4eca8f5dac99346dc0fbfea6a3a6e07ad9c5
+size 598520
diff --git a/phivenv/Scripts/tiny-agents.exe b/phivenv/Scripts/tiny-agents.exe
new file mode 100644
index 0000000000000000000000000000000000000000..a1b3b93eda7d902dbbfc04968258c03fe26962d3
--- /dev/null
+++ b/phivenv/Scripts/tiny-agents.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403e8c6b78d5c32b5a6763e4258013ff32284fcf08fb30d808fd8036c6cb2ce6
+size 106362
diff --git a/phivenv/Scripts/torchfrtrace.exe b/phivenv/Scripts/torchfrtrace.exe
new file mode 100644
index 0000000000000000000000000000000000000000..0f027148e9f71b11d5cae496e4f47ed886d274f2
--- /dev/null
+++ b/phivenv/Scripts/torchfrtrace.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8523d09ed1107f4e77c9b04c0b7b9681c7c474388b446ce46e112c9de14d1ff5
+size 106360
diff --git a/phivenv/Scripts/torchrun.exe b/phivenv/Scripts/torchrun.exe
new file mode 100644
index 0000000000000000000000000000000000000000..21525359763b815ff74faf4581b084356beefadc
--- /dev/null
+++ b/phivenv/Scripts/torchrun.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9aeb0d13901603e0ce4f5d5942173599cf18a101f46857c0ab2f4d197cef808
+size 106351
diff --git a/phivenv/Scripts/tqdm.exe b/phivenv/Scripts/tqdm.exe
new file mode 100644
index 0000000000000000000000000000000000000000..239b79fd64a36e61dc577da4cfe0e179002aa56d
--- /dev/null
+++ b/phivenv/Scripts/tqdm.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9726e8dd4c92b6543279dc4a25af76a408a61461773c7bd71450d790ff4ea119
+size 106338
diff --git a/phivenv/Scripts/transformers-cli.exe b/phivenv/Scripts/transformers-cli.exe
new file mode 100644
index 0000000000000000000000000000000000000000..2e97d7ae52aaf143496834ac86bc35f73d4ee875
--- /dev/null
+++ b/phivenv/Scripts/transformers-cli.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce2b371a0475c98568c3c65e576e3ffe7ae576d876bd01dd26762ed103c853e0
+size 106376
diff --git a/phivenv/Scripts/transformers.exe b/phivenv/Scripts/transformers.exe
new file mode 100644
index 0000000000000000000000000000000000000000..63b9454fded02c649849536f46fbca0fe265596a
--- /dev/null
+++ b/phivenv/Scripts/transformers.exe
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d5406fe4a25c3f71118850c8e2c69babb2f4ffb3144d8f3bec3f612297217ef
+size 106368
diff --git a/phivenv/share/man/man1/isympy.1 b/phivenv/share/man/man1/isympy.1
new file mode 100644
index 0000000000000000000000000000000000000000..0ff966158a28c5ad1a6cd954e454842b25fdd999
--- /dev/null
+++ b/phivenv/share/man/man1/isympy.1
@@ -0,0 +1,188 @@
+'\" -*- coding: us-ascii -*-
+.if \n(.g .ds T< \\FC
+.if \n(.g .ds T> \\F[\n[.fam]]
+.de URL
+\\$2 \(la\\$1\(ra\\$3
+..
+.if \n(.g .mso www.tmac
+.TH isympy 1 2007-10-8 "" ""
+.SH NAME
+isympy \- interactive shell for SymPy
+.SH SYNOPSIS
+'nh
+.fi
+.ad l
+\fBisympy\fR \kx
+.if (\nx>(\n(.l/2)) .nr x (\n(.l/5)
+'in \n(.iu+\nxu
+[\fB-c\fR | \fB--console\fR] [\fB-p\fR ENCODING | \fB--pretty\fR ENCODING] [\fB-t\fR TYPE | \fB--types\fR TYPE] [\fB-o\fR ORDER | \fB--order\fR ORDER] [\fB-q\fR | \fB--quiet\fR] [\fB-d\fR | \fB--doctest\fR] [\fB-C\fR | \fB--no-cache\fR] [\fB-a\fR | \fB--auto\fR] [\fB-D\fR | \fB--debug\fR] [
+-- | PYTHONOPTIONS]
+'in \n(.iu-\nxu
+.ad b
+'hy
+'nh
+.fi
+.ad l
+\fBisympy\fR \kx
+.if (\nx>(\n(.l/2)) .nr x (\n(.l/5)
+'in \n(.iu+\nxu
+[
+{\fB-h\fR | \fB--help\fR}
+|
+{\fB-v\fR | \fB--version\fR}
+]
+'in \n(.iu-\nxu
+.ad b
+'hy
+.SH DESCRIPTION
+isympy is a Python shell for SymPy. It is just a normal python shell
+(ipython shell if you have the ipython package installed) that executes
+the following commands so that you don't have to:
+.PP
+.nf
+\*(T<
+>>> from __future__ import division
+>>> from sympy import *
+>>> x, y, z = symbols("x,y,z")
+>>> k, m, n = symbols("k,m,n", integer=True)
+ \*(T>
+.fi
+.PP
+So starting isympy is equivalent to starting python (or ipython) and
+executing the above commands by hand. It is intended for easy and quick
+experimentation with SymPy. For more complicated programs, it is recommended
+to write a script and import things explicitly (using the "from sympy
+import sin, log, Symbol, ..." idiom).
+.SH OPTIONS
+.TP
+\*(T<\fB\-c \fR\*(T>\fISHELL\fR, \*(T<\fB\-\-console=\fR\*(T>\fISHELL\fR
+Use the specified shell (python or ipython) as
+console backend instead of the default one (ipython
+if present or python otherwise).
+
+Example: isympy -c python
+
+\fISHELL\fR could be either
+\&'ipython' or 'python'
+.TP
+\*(T<\fB\-p \fR\*(T>\fIENCODING\fR, \*(T<\fB\-\-pretty=\fR\*(T>\fIENCODING\fR
+Setup pretty printing in SymPy. By default, the most pretty, unicode
+printing is enabled (if the terminal supports it). You can use less
+pretty ASCII printing instead or no pretty printing at all.
+
+Example: isympy -p no
+
+\fIENCODING\fR must be one of 'unicode',
+\&'ascii' or 'no'.
+.TP
+\*(T<\fB\-t \fR\*(T>\fITYPE\fR, \*(T<\fB\-\-types=\fR\*(T>\fITYPE\fR
+Setup the ground types for the polys. By default, gmpy ground types
+are used if gmpy2 or gmpy is installed, otherwise it falls back to python
+ground types, which are a little bit slower. You can manually
+choose python ground types even if gmpy is installed (e.g., for testing purposes).
+
+Note that sympy ground types are not supported, and should be used
+only for experimental purposes.
+
+Note that the gmpy1 ground type is primarily intended for testing; it the
+use of gmpy even if gmpy2 is available.
+
+This is the same as setting the environment variable
+SYMPY_GROUND_TYPES to the given ground type (e.g.,
+SYMPY_GROUND_TYPES='gmpy')
+
+The ground types can be determined interactively from the variable
+sympy.polys.domains.GROUND_TYPES inside the isympy shell itself.
+
+Example: isympy -t python
+
+\fITYPE\fR must be one of 'gmpy',
+\&'gmpy1' or 'python'.
+.TP
+\*(T<\fB\-o \fR\*(T>\fIORDER\fR, \*(T<\fB\-\-order=\fR\*(T>\fIORDER\fR
+Setup the ordering of terms for printing. The default is lex, which
+orders terms lexicographically (e.g., x**2 + x + 1). You can choose
+other orderings, such as rev-lex, which will use reverse
+lexicographic ordering (e.g., 1 + x + x**2).
+
+Note that for very large expressions, ORDER='none' may speed up
+printing considerably, with the tradeoff that the order of the terms
+in the printed expression will have no canonical order
+
+Example: isympy -o rev-lax
+
+\fIORDER\fR must be one of 'lex', 'rev-lex', 'grlex',
+\&'rev-grlex', 'grevlex', 'rev-grevlex', 'old', or 'none'.
+.TP
+\*(T<\fB\-q\fR\*(T>, \*(T<\fB\-\-quiet\fR\*(T>
+Print only Python's and SymPy's versions to stdout at startup, and nothing else.
+.TP
+\*(T<\fB\-d\fR\*(T>, \*(T<\fB\-\-doctest\fR\*(T>
+Use the same format that should be used for doctests. This is
+equivalent to '\fIisympy -c python -p no\fR'.
+.TP
+\*(T<\fB\-C\fR\*(T>, \*(T<\fB\-\-no\-cache\fR\*(T>
+Disable the caching mechanism. Disabling the cache may slow certain
+operations down considerably. This is useful for testing the cache,
+or for benchmarking, as the cache can result in deceptive benchmark timings.
+
+This is the same as setting the environment variable SYMPY_USE_CACHE
+to 'no'.
+.TP
+\*(T<\fB\-a\fR\*(T>, \*(T<\fB\-\-auto\fR\*(T>
+Automatically create missing symbols. Normally, typing a name of a
+Symbol that has not been instantiated first would raise NameError,
+but with this option enabled, any undefined name will be
+automatically created as a Symbol. This only works in IPython 0.11.
+
+Note that this is intended only for interactive, calculator style
+usage. In a script that uses SymPy, Symbols should be instantiated
+at the top, so that it's clear what they are.
+
+This will not override any names that are already defined, which
+includes the single character letters represented by the mnemonic
+QCOSINE (see the "Gotchas and Pitfalls" document in the
+documentation). You can delete existing names by executing "del
+name" in the shell itself. You can see if a name is defined by typing
+"'name' in globals()".
+
+The Symbols that are created using this have default assumptions.
+If you want to place assumptions on symbols, you should create them
+using symbols() or var().
+
+Finally, this only works in the top level namespace. So, for
+example, if you define a function in isympy with an undefined
+Symbol, it will not work.
+.TP
+\*(T<\fB\-D\fR\*(T>, \*(T<\fB\-\-debug\fR\*(T>
+Enable debugging output. This is the same as setting the
+environment variable SYMPY_DEBUG to 'True'. The debug status is set
+in the variable SYMPY_DEBUG within isympy.
+.TP
+-- \fIPYTHONOPTIONS\fR
+These options will be passed on to \fIipython (1)\fR shell.
+Only supported when ipython is being used (standard python shell not supported).
+
+Two dashes (--) are required to separate \fIPYTHONOPTIONS\fR
+from the other isympy options.
+
+For example, to run iSymPy without startup banner and colors:
+
+isympy -q -c ipython -- --colors=NoColor
+.TP
+\*(T<\fB\-h\fR\*(T>, \*(T<\fB\-\-help\fR\*(T>
+Print help output and exit.
+.TP
+\*(T<\fB\-v\fR\*(T>, \*(T<\fB\-\-version\fR\*(T>
+Print isympy version information and exit.
+.SH FILES
+.TP
+\*(T<\fI${HOME}/.sympy\-history\fR\*(T>
+Saves the history of commands when using the python
+shell as backend.
+.SH BUGS
+The upstreams BTS can be found at \(lahttps://github.com/sympy/sympy/issues\(ra
+Please report all bugs that you find in there, this will help improve
+the overall quality of SymPy.
+.SH "SEE ALSO"
+\fBipython\fR(1), \fBpython\fR(1)
+ 
+ 
+ 
+ 
+ 
+
+
+
+