nem sei pq tantos arquivos

This commit is contained in:
2025-02-11 11:07:58 -03:00
parent 66fb4eb17b
commit 2da09a8a25
1841 changed files with 115867 additions and 77478 deletions

View File

@@ -3,9 +3,8 @@ import hashlib
import logging
import os
from types import TracebackType
from typing import Dict, Generator, Optional, Set, Type, Union
from typing import Dict, Generator, Optional, Type, Union
from pip._internal.models.link import Link
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.temp_dir import TempDirectory
@@ -51,10 +50,22 @@ def get_build_tracker() -> Generator["BuildTracker", None, None]:
yield tracker
class TrackerId(str):
"""Uniquely identifying string provided to the build tracker."""
class BuildTracker:
"""Ensure that an sdist cannot request itself as a setup requirement.
When an sdist is prepared, it identifies its setup requirements in the
context of ``BuildTracker.track()``. If a requirement shows up recursively, this
raises an exception.
This stops fork bombs embedded in malicious packages."""
def __init__(self, root: str) -> None:
self._root = root
self._entries: Set[InstallRequirement] = set()
self._entries: Dict[TrackerId, InstallRequirement] = {}
logger.debug("Created build tracker: %s", self._root)
def __enter__(self) -> "BuildTracker":
@@ -69,16 +80,15 @@ class BuildTracker:
) -> None:
self.cleanup()
def _entry_path(self, link: Link) -> str:
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
def _entry_path(self, key: TrackerId) -> str:
hashed = hashlib.sha224(key.encode()).hexdigest()
return os.path.join(self._root, hashed)
def add(self, req: InstallRequirement) -> None:
def add(self, req: InstallRequirement, key: TrackerId) -> None:
"""Add an InstallRequirement to build tracking."""
assert req.link
# Get the file to write information about this requirement.
entry_path = self._entry_path(req.link)
entry_path = self._entry_path(key)
# Try reading from the file. If it exists and can be read from, a build
# is already in progress, so a LookupError is raised.
@@ -88,37 +98,41 @@ class BuildTracker:
except FileNotFoundError:
pass
else:
message = "{} is already being built: {}".format(req.link, contents)
message = f"{req.link} is already being built: {contents}"
raise LookupError(message)
# If we're here, req should really not be building already.
assert req not in self._entries
assert key not in self._entries
# Start tracking this requirement.
with open(entry_path, "w", encoding="utf-8") as fp:
fp.write(str(req))
self._entries.add(req)
self._entries[key] = req
logger.debug("Added %s to build tracker %r", req, self._root)
def remove(self, req: InstallRequirement) -> None:
def remove(self, req: InstallRequirement, key: TrackerId) -> None:
"""Remove an InstallRequirement from build tracking."""
assert req.link
# Delete the created file and the corresponding entries.
os.unlink(self._entry_path(req.link))
self._entries.remove(req)
# Delete the created file and the corresponding entry.
os.unlink(self._entry_path(key))
del self._entries[key]
logger.debug("Removed %s from build tracker %r", req, self._root)
def cleanup(self) -> None:
for req in set(self._entries):
self.remove(req)
for key, req in list(self._entries.items()):
self.remove(req, key)
logger.debug("Removed build tracker: %r", self._root)
@contextlib.contextmanager
def track(self, req: InstallRequirement) -> Generator[None, None, None]:
self.add(req)
def track(self, req: InstallRequirement, key: str) -> Generator[None, None, None]:
"""Ensure that `key` cannot install itself as a setup requirement.
:raises LookupError: If `key` was already provided in a parent invocation of
the context introduced by this method."""
tracker_id = TrackerId(key)
self.add(req, tracker_id)
yield
self.remove(req)
self.remove(req, tracker_id)

View File

@@ -38,4 +38,5 @@ def generate_editable_metadata(
except InstallationSubprocessError as error:
raise MetadataGenerationFailed(package_details=details) from error
assert distinfo_dir is not None
return os.path.join(metadata_dir, distinfo_dir)

View File

@@ -27,7 +27,7 @@ def _find_egg_info(directory: str) -> str:
if len(filenames) > 1:
raise InstallationError(
"More than one .egg-info directory found in {}".format(directory)
f"More than one .egg-info directory found in {directory}"
)
return os.path.join(directory, filenames[0])

View File

@@ -40,16 +40,16 @@ def get_legacy_build_wheel_path(
# Sort for determinism.
names = sorted(names)
if not names:
msg = ("Legacy build of wheel for {!r} created no files.\n").format(name)
msg = f"Legacy build of wheel for {name!r} created no files.\n"
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return None
if len(names) > 1:
msg = (
"Legacy build of wheel for {!r} created more than one file.\n"
"Filenames (choosing first): {}\n"
).format(name, names)
f"Legacy build of wheel for {name!r} created more than one file.\n"
f"Filenames (choosing first): {names}\n"
)
msg += format_command_result(command_args, command_output)
logger.warning(msg)

View File

@@ -2,28 +2,44 @@
"""
import logging
from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple
from contextlib import suppress
from email.parser import Parser
from functools import reduce
from typing import (
Callable,
Dict,
FrozenSet,
Generator,
Iterable,
List,
NamedTuple,
Optional,
Set,
Tuple,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.tags import Tag, parse_tag
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.base import DistributionVersion
from pip._internal.metadata.base import BaseDistribution
from pip._internal.req.req_install import InstallRequirement
logger = logging.getLogger(__name__)
class PackageDetails(NamedTuple):
version: DistributionVersion
version: Version
dependencies: List[Requirement]
# Shorthands
PackageSet = Dict[NormalizedName, PackageDetails]
Missing = Tuple[NormalizedName, Requirement]
Conflicting = Tuple[NormalizedName, DistributionVersion, Requirement]
Conflicting = Tuple[NormalizedName, Version, Requirement]
MissingDict = Dict[NormalizedName, List[Missing]]
ConflictingDict = Dict[NormalizedName, List[Conflicting]]
@@ -43,7 +59,7 @@ def create_package_set_from_installed() -> Tuple[PackageSet, bool]:
package_set[name] = PackageDetails(dist.version, dependencies)
except (OSError, ValueError) as e:
# Don't crash on unreadable or broken metadata.
logger.warning("Error parsing requirements for %s: %s", name, e)
logger.warning("Error parsing dependencies of %s: %s", name, e)
problems = True
return package_set, problems
@@ -113,6 +129,22 @@ def check_install_conflicts(to_install: List[InstallRequirement]) -> ConflictDet
)
def check_unsupported(
packages: Iterable[BaseDistribution],
supported_tags: Iterable[Tag],
) -> Generator[BaseDistribution, None, None]:
for p in packages:
with suppress(FileNotFoundError):
wheel_file = p.read_text("WHEEL")
wheel_tags: FrozenSet[Tag] = reduce(
frozenset.union,
map(parse_tag, Parser().parsestr(wheel_file).get_all("Tag", [])),
frozenset(),
)
if wheel_tags.isdisjoint(supported_tags):
yield p
def _simulate_installation_of(
to_install: List[InstallRequirement], package_set: PackageSet
) -> Set[NormalizedName]:

View File

@@ -1,10 +1,11 @@
import collections
import logging
import os
from dataclasses import dataclass, field
from typing import Container, Dict, Generator, Iterable, List, NamedTuple, Optional, Set
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import InvalidVersion
from pip._internal.exceptions import BadCommand, InstallationError
from pip._internal.metadata import BaseDistribution, get_environment
@@ -145,9 +146,13 @@ def freeze(
def _format_as_name_version(dist: BaseDistribution) -> str:
if isinstance(dist.version, Version):
return f"{dist.raw_name}=={dist.version}"
return f"{dist.raw_name}==={dist.version}"
try:
dist_version = dist.version
except InvalidVersion:
# legacy version
return f"{dist.raw_name}==={dist.raw_version}"
else:
return f"{dist.raw_name}=={dist_version}"
def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
@@ -216,19 +221,16 @@ def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
)
@dataclass(frozen=True)
class FrozenRequirement:
def __init__(
self,
name: str,
req: str,
editable: bool,
comments: Iterable[str] = (),
) -> None:
self.name = name
self.canonical_name = canonicalize_name(name)
self.req = req
self.editable = editable
self.comments = comments
name: str
req: str
editable: bool
comments: Iterable[str] = field(default_factory=tuple)
@property
def canonical_name(self) -> NormalizedName:
return canonicalize_name(self.name)
@classmethod
def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement":

View File

@@ -1,7 +1,8 @@
"""Legacy editable installation process, i.e. `setup.py develop`.
"""
import logging
from typing import List, Optional, Sequence
from typing import Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.utils.logging import indent_log
@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
def install_editable(
install_options: List[str],
*,
global_options: Sequence[str],
prefix: Optional[str],
home: Optional[str],
@@ -31,7 +32,6 @@ def install_editable(
args = make_setuptools_develop_args(
setup_py_path,
global_options=global_options,
install_options=install_options,
no_user_config=isolated,
prefix=prefix,
home=home,

View File

@@ -1,120 +0,0 @@
"""Legacy installation process, i.e. `setup.py install`.
"""
import logging
import os
from typing import List, Optional, Sequence
from pip._internal.build_env import BuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.locations.base import change_root
from pip._internal.models.scheme import Scheme
from pip._internal.utils.misc import ensure_dir
from pip._internal.utils.setuptools_build import make_setuptools_install_args
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory
logger = logging.getLogger(__name__)
def write_installed_files_from_setuptools_record(
record_lines: List[str],
root: Optional[str],
req_description: str,
) -> None:
def prepend_root(path: str) -> str:
if root is None or not os.path.isabs(path):
return path
else:
return change_root(root, path)
for line in record_lines:
directory = os.path.dirname(line)
if directory.endswith(".egg-info"):
egg_info_dir = prepend_root(directory)
break
else:
message = (
"{} did not indicate that it installed an "
".egg-info directory. Only setup.py projects "
"generating .egg-info directories are supported."
).format(req_description)
raise InstallationError(message)
new_lines = []
for line in record_lines:
filename = line.strip()
if os.path.isdir(filename):
filename += os.path.sep
new_lines.append(os.path.relpath(prepend_root(filename), egg_info_dir))
new_lines.sort()
ensure_dir(egg_info_dir)
inst_files_path = os.path.join(egg_info_dir, "installed-files.txt")
with open(inst_files_path, "w") as f:
f.write("\n".join(new_lines) + "\n")
def install(
install_options: List[str],
global_options: Sequence[str],
root: Optional[str],
home: Optional[str],
prefix: Optional[str],
use_user_site: bool,
pycompile: bool,
scheme: Scheme,
setup_py_path: str,
isolated: bool,
req_name: str,
build_env: BuildEnvironment,
unpacked_source_directory: str,
req_description: str,
) -> bool:
header_dir = scheme.headers
with TempDirectory(kind="record") as temp_dir:
try:
record_filename = os.path.join(temp_dir.path, "install-record.txt")
install_args = make_setuptools_install_args(
setup_py_path,
global_options=global_options,
install_options=install_options,
record_filename=record_filename,
root=root,
prefix=prefix,
header_dir=header_dir,
home=home,
use_user_site=use_user_site,
no_user_config=isolated,
pycompile=pycompile,
)
runner = runner_with_spinner_message(
f"Running setup.py install for {req_name}"
)
with build_env:
runner(
cmd=install_args,
cwd=unpacked_source_directory,
)
if not os.path.exists(record_filename):
logger.debug("Record file %s not found", record_filename)
# Signal to the caller that we didn't install the new package
return False
except Exception as e:
# Signal to the caller that we didn't install the new package
raise LegacyInstallFailure(package_details=req_name) from e
# At this point, we have successfully installed the requirement.
# We intentionally do not use any encoding to read the file because
# setuptools writes the file using distutils.file_util.write_file,
# which does not specify an encoding.
with open(record_filename) as f:
record_lines = f.read().splitlines()
write_installed_files_from_setuptools_record(record_lines, root, req_description)
return True

View File

@@ -28,6 +28,7 @@ from typing import (
List,
NewType,
Optional,
Protocol,
Sequence,
Set,
Tuple,
@@ -50,7 +51,7 @@ from pip._internal.metadata import (
from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file, partition
from pip._internal.utils.misc import StreamWrapper, ensure_dir, hash_file, partition
from pip._internal.utils.unpacking import (
current_umask,
is_within_directory,
@@ -60,7 +61,6 @@ from pip._internal.utils.unpacking import (
from pip._internal.utils.wheel import parse_wheel
if TYPE_CHECKING:
from typing import Protocol
class File(Protocol):
src_record_path: "RecordPath"
@@ -143,16 +143,18 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
# We don't want to warn for directories that are on PATH.
not_warn_dirs = [
os.path.normcase(i).rstrip(os.sep)
os.path.normcase(os.path.normpath(i)).rstrip(os.sep)
for i in os.environ.get("PATH", "").split(os.pathsep)
]
# If an executable sits with sys.executable, we don't warn for it.
# This covers the case of venv invocations without activating the venv.
not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable)))
not_warn_dirs.append(
os.path.normcase(os.path.normpath(os.path.dirname(sys.executable)))
)
warn_for: Dict[str, Set[str]] = {
parent_dir: scripts
for parent_dir, scripts in grouped_by_dir.items()
if os.path.normcase(parent_dir) not in not_warn_dirs
if os.path.normcase(os.path.normpath(parent_dir)) not in not_warn_dirs
}
if not warn_for:
return None
@@ -162,16 +164,14 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
for parent_dir, dir_scripts in warn_for.items():
sorted_scripts: List[str] = sorted(dir_scripts)
if len(sorted_scripts) == 1:
start_text = "script {} is".format(sorted_scripts[0])
start_text = f"script {sorted_scripts[0]} is"
else:
start_text = "scripts {} are".format(
", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1]
)
msg_lines.append(
"The {} installed in '{}' which is not on PATH.".format(
start_text, parent_dir
)
f"The {start_text} installed in '{parent_dir}' which is not on PATH."
)
last_line_fmt = (
@@ -265,9 +265,9 @@ def get_csv_rows_for_installed(
path = _fs_to_record_path(f, lib_dir)
digest, length = rehash(f)
installed_rows.append((path, digest, length))
for installed_record_path in installed.values():
installed_rows.append((installed_record_path, "", ""))
return installed_rows
return installed_rows + [
(installed_record_path, "", "") for installed_record_path in installed.values()
]
def get_console_script_specs(console: Dict[str, str]) -> List[str]:
@@ -288,17 +288,15 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
# the wheel metadata at build time, and so if the wheel is installed with
# a *different* version of Python the entry points will be wrong. The
# correct fix for this is to enhance the metadata to be able to describe
# such versioned entry points, but that won't happen till Metadata 2.0 is
# available.
# In the meantime, projects using versioned entry points will either have
# such versioned entry points.
# Currently, projects using versioned entry points will either have
# incorrect versioned entry points, or they will not be able to distribute
# "universal" wheels (i.e., they will need a wheel per Python version).
#
# Because setuptools and pip are bundled with _ensurepip and virtualenv,
# we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
# we need to use universal wheels. As a workaround, we
# override the versioned entry points in the wheel and generate the
# correct ones. This code is purely a short-term measure until Metadata 2.0
# is available.
# correct ones.
#
# To add the level of hack in this section of code, in order to support
# ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
@@ -319,9 +317,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
scripts_to_generate.append("pip = " + pip_script)
if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
scripts_to_generate.append(
"pip{} = {}".format(sys.version_info[0], pip_script)
)
scripts_to_generate.append(f"pip{sys.version_info[0]} = {pip_script}")
scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}")
# Delete any other versioned pip entry points
@@ -334,9 +330,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
scripts_to_generate.append("easy_install = " + easy_install_script)
scripts_to_generate.append(
"easy_install-{} = {}".format(
get_major_minor_version(), easy_install_script
)
f"easy_install-{get_major_minor_version()} = {easy_install_script}"
)
# Delete any other versioned easy_install entry points
easy_install_ep = [
@@ -364,12 +358,6 @@ class ZipBackedFile:
return self._zip_file.getinfo(self.src_record_path)
def save(self) -> None:
# directory creation is lazy and after file filtering
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
parent_dir = os.path.dirname(self.dest_path)
ensure_dir(parent_dir)
# When we open the output file below, any existing file is truncated
# before we start writing the new contents. This is fine in most
# cases, but can cause a segfault if pip has loaded a shared
@@ -383,9 +371,13 @@ class ZipBackedFile:
zipinfo = self._getinfo()
with self._zip_file.open(zipinfo) as f:
with open(self.dest_path, "wb") as dest:
shutil.copyfileobj(f, dest)
# optimization: the file is created by open(),
# skip the decompression when there is 0 bytes to decompress.
with open(self.dest_path, "wb") as dest:
if zipinfo.file_size > 0:
with self._zip_file.open(zipinfo) as f:
blocksize = min(zipinfo.file_size, 1024 * 1024)
shutil.copyfileobj(f, dest, blocksize)
if zip_item_is_executable(zipinfo):
set_extracted_file_to_default_mode_plus_executable(self.dest_path)
@@ -406,10 +398,10 @@ class ScriptFile:
class MissingCallableSuffix(InstallationError):
def __init__(self, entry_point: str) -> None:
super().__init__(
"Invalid script entry point: {} - A callable "
f"Invalid script entry point: {entry_point} - A callable "
"suffix is required. Cf https://packaging.python.org/"
"specifications/entry-points/#use-for-scripts for more "
"information.".format(entry_point)
"information."
)
@@ -427,7 +419,7 @@ class PipScriptMaker(ScriptMaker):
return super().make(specification, options)
def _install_wheel(
def _install_wheel( # noqa: C901, PLR0915 function is too long
name: str,
wheel_zip: ZipFile,
wheel_path: str,
@@ -511,9 +503,9 @@ def _install_wheel(
_, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2)
except ValueError:
message = (
"Unexpected file in {}: {!r}. .data directory contents"
" should be named like: '<scheme key>/<path>'."
).format(wheel_path, record_path)
f"Unexpected file in {wheel_path}: {record_path!r}. .data directory"
" contents should be named like: '<scheme key>/<path>'."
)
raise InstallationError(message)
try:
@@ -521,10 +513,11 @@ def _install_wheel(
except KeyError:
valid_scheme_keys = ", ".join(sorted(scheme_paths))
message = (
"Unknown scheme key used in {}: {} (for file {!r}). .data"
" directory contents should be in subdirectories named"
" with a valid scheme key ({})"
).format(wheel_path, scheme_key, record_path, valid_scheme_keys)
f"Unknown scheme key used in {wheel_path}: {scheme_key} "
f"(for file {record_path!r}). .data directory contents "
f"should be in subdirectories named with a valid scheme "
f"key ({valid_scheme_keys})"
)
raise InstallationError(message)
dest_path = os.path.join(scheme_path, dest_subpath)
@@ -585,7 +578,15 @@ def _install_wheel(
script_scheme_files = map(ScriptFile, script_scheme_files)
files = chain(files, script_scheme_files)
existing_parents = set()
for file in files:
# directory creation is lazy and after file filtering
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
parent_dir = os.path.dirname(file.dest_path)
if parent_dir not in existing_parents:
ensure_dir(parent_dir)
existing_parents.add(parent_dir)
file.save()
record_installed(file.src_record_path, file.dest_path, file.changed)
@@ -608,7 +609,9 @@ def _install_wheel(
# Compile all of the pyc files for the installed files
if pycompile:
with captured_stdout() as stdout:
with contextlib.redirect_stdout(
StreamWrapper.from_stream(sys.stdout)
) as stdout:
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
for path in pyc_source_file_paths():
@@ -710,7 +713,7 @@ def req_error_context(req_description: str) -> Generator[None, None, None]:
try:
yield
except InstallationError as e:
message = "For req: {}. {}".format(req_description, e.args[0])
message = f"For req: {req_description}. {e.args[0]}"
raise InstallationError(message) from e

View File

@@ -4,10 +4,11 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import mimetypes
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from pip._vendor.packaging.utils import canonicalize_name
@@ -21,7 +22,6 @@ from pip._internal.exceptions import (
InstallationError,
MetadataInconsistent,
NetworkConnectionError,
PreviousBuildDirError,
VcsHashUnsupported,
)
from pip._internal.index.package_finder import PackageFinder
@@ -37,6 +37,7 @@ from pip._internal.network.lazy_wheel import (
from pip._internal.network.session import PipSession
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils._log import getLogger
from pip._internal.utils.direct_url_helpers import (
direct_url_for_editable,
direct_url_from_link,
@@ -47,13 +48,13 @@ from pip._internal.utils.misc import (
display_path,
hash_file,
hide_url,
is_installable_dir,
redact_auth_from_requirement,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
logger = getLogger(__name__)
def _get_prepared_distribution(
@@ -65,10 +66,12 @@ def _get_prepared_distribution(
) -> BaseDistribution:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
with build_tracker.track(req):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
tracker_id = abstract_dist.build_tracker_id
if tracker_id is not None:
with build_tracker.track(req, tracker_id):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
return abstract_dist.get_metadata_distribution()
@@ -78,13 +81,14 @@ def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
@dataclass
class File:
def __init__(self, path: str, content_type: Optional[str]) -> None:
self.path = path
if content_type is None:
self.content_type = mimetypes.guess_type(path)[0]
else:
self.content_type = content_type
path: str
content_type: Optional[str] = None
def __post_init__(self) -> None:
if self.content_type is None:
self.content_type = mimetypes.guess_type(self.path)[0]
def get_http_url(
@@ -179,7 +183,10 @@ def unpack_url(
def _check_download_dir(
link: Link, download_dir: str, hashes: Optional[Hashes]
link: Link,
download_dir: str,
hashes: Optional[Hashes],
warn_on_hash_mismatch: bool = True,
) -> Optional[str]:
"""Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
@@ -195,10 +202,11 @@ def _check_download_dir(
try:
hashes.check_against_path(download_path)
except HashMismatch:
logger.warning(
"Previously-downloaded file %s has bad hash. Re-downloading.",
download_path,
)
if warn_on_hash_mismatch:
logger.warning(
"Previously-downloaded file %s has bad hash. Re-downloading.",
download_path,
)
os.unlink(download_path)
return None
return download_path
@@ -222,6 +230,7 @@ class RequirementPreparer:
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
legacy_resolver: bool,
) -> None:
super().__init__()
@@ -255,6 +264,9 @@ class RequirementPreparer:
# How verbose should underlying tooling be?
self.verbosity = verbosity
# Are we using the legacy resolver?
self.legacy_resolver = legacy_resolver
# Memoized downloaded files, as mapping of url: path.
self._downloaded: Dict[str, str] = {}
@@ -263,18 +275,28 @@ class RequirementPreparer:
def _log_preparing_link(self, req: InstallRequirement) -> None:
"""Provide context for the requirement being prepared."""
if req.link.is_file and not req.original_link_is_in_wheel_cache:
if req.link.is_file and not req.is_wheel_from_cache:
message = "Processing %s"
information = str(display_path(req.link.file_path))
else:
message = "Collecting %s"
information = str(req.req or req)
information = redact_auth_from_requirement(req.req) if req.req else str(req)
# If we used req.req, inject requirement source if available (this
# would already be included if we used req directly)
if req.req and req.comes_from:
if isinstance(req.comes_from, str):
comes_from: Optional[str] = req.comes_from
else:
comes_from = req.comes_from.from_path()
if comes_from:
information += f" (from {comes_from})"
if (message, information) != self._previous_requirement_header:
self._previous_requirement_header = (message, information)
logger.info(message, information)
if req.original_link_is_in_wheel_cache:
if req.is_wheel_from_cache:
with indent_log():
logger.info("Using cached %s", req.link.filename)
@@ -299,21 +321,7 @@ class RequirementPreparer:
autodelete=True,
parallel_builds=parallel_builds,
)
# If a checkout exists, it's unwise to keep going. version
# inconsistencies are logged later, but do not fail the
# installation.
# FIXME: this won't upgrade when there's an existing
# package unpacked in `req.source_dir`
# TODO: this check is now probably dead code
if is_installable_dir(req.source_dir):
raise PreviousBuildDirError(
"pip can't proceed with requirements '{}' due to a"
"pre-existing build directory ({}). This is likely "
"due to a previous installation that failed . pip is "
"being responsible and not assuming it can delete this. "
"Please delete it and try again.".format(req, req.source_dir)
)
req.ensure_pristine_source_checkout()
def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
# By the time this is called, the requirement's link should have
@@ -338,7 +346,7 @@ class RequirementPreparer:
# a surprising hash mismatch in the future.
# file:/// URLs aren't pinnable, so don't complain about them
# not being pinned.
if req.original_link is None and not req.is_pinned:
if not req.is_direct and not req.is_pinned:
raise HashUnpinned()
# If known-good hashes are missing for this requirement,
@@ -351,6 +359,11 @@ class RequirementPreparer:
self,
req: InstallRequirement,
) -> Optional[BaseDistribution]:
if self.legacy_resolver:
logger.debug(
"Metadata-only fetching is not used in the legacy resolver",
)
return None
if self.require_hashes:
logger.debug(
"Metadata-only fetching is not used as hash checking is required",
@@ -371,7 +384,7 @@ class RequirementPreparer:
if metadata_link is None:
return None
assert req.req is not None
logger.info(
logger.verbose(
"Obtaining dependency information for %s from %s",
req.req,
metadata_link,
@@ -396,7 +409,7 @@ class RequirementPreparer:
# NB: raw_name will fall back to the name from the install requirement if
# the Name: field is not present, but it's noted in the raw_name docstring
# that that should NEVER happen anyway.
if metadata_dist.raw_name != req.req.name:
if canonicalize_name(metadata_dist.raw_name) != canonicalize_name(req.req.name):
raise MetadataInconsistent(
req, "Name", req.req.name, metadata_dist.raw_name
)
@@ -456,7 +469,19 @@ class RequirementPreparer:
for link, (filepath, _) in batch_download:
logger.debug("Downloading link %s to %s", link, filepath)
req = links_to_fully_download[link]
# Record the downloaded file path so wheel reqs can extract a Distribution
# in .get_dist().
req.local_file_path = filepath
# Record that the file is downloaded so we don't do it again in
# _prepare_linked_requirement().
self._downloaded[req.link.url] = filepath
# If this is an sdist, we need to unpack it after downloading, but the
# .source_dir won't be set up until we are in _prepare_linked_requirement().
# Add the downloaded archive to the install requirement to unpack after
# preparing the source dir.
if not req.is_wheel:
req.needs_unpacked_archive(Path(filepath))
# This step is necessary to ensure all lazy wheels are processed
# successfully by the 'download', 'wheel', and 'install' commands.
@@ -475,7 +500,18 @@ class RequirementPreparer:
file_path = None
if self.download_dir is not None and req.link.is_wheel:
hashes = self._get_linked_req_hashes(req)
file_path = _check_download_dir(req.link, self.download_dir, hashes)
file_path = _check_download_dir(
req.link,
self.download_dir,
hashes,
# When a locally built wheel has been found in cache, we don't warn
# about re-downloading when the already downloaded wheel hash does
# not match. This is because the hash must be checked against the
# original link, not the cached link. It that case the already
# downloaded file will be removed and re-fetched from cache (which
# implies a hash check against the cache entry's origin.json).
warn_on_hash_mismatch=not req.is_wheel_from_cache,
)
if file_path is not None:
# The file is already available, so mark it as downloaded
@@ -526,9 +562,35 @@ class RequirementPreparer:
assert req.link
link = req.link
self._ensure_link_req_src_dir(req, parallel_builds)
hashes = self._get_linked_req_hashes(req)
if hashes and req.is_wheel_from_cache:
assert req.download_info is not None
assert link.is_wheel
assert link.is_file
# We need to verify hashes, and we have found the requirement in the cache
# of locally built wheels.
if (
isinstance(req.download_info.info, ArchiveInfo)
and req.download_info.info.hashes
and hashes.has_one_of(req.download_info.info.hashes)
):
# At this point we know the requirement was built from a hashable source
# artifact, and we verified that the cache entry's hash of the original
# artifact matches one of the hashes we expect. We don't verify hashes
# against the cached wheel, because the wheel is not the original.
hashes = None
else:
logger.warning(
"The hashes of the source archive found in cache entry "
"don't match, ignoring cached built wheel "
"and re-downloading source."
)
req.link = req.cached_wheel_source_link
link = req.link
self._ensure_link_req_src_dir(req, parallel_builds)
if link.is_existing_dir():
local_file = None
elif link.url not in self._downloaded:
@@ -543,8 +605,8 @@ class RequirementPreparer:
)
except NetworkConnectionError as exc:
raise InstallationError(
"Could not install requirement {} because of HTTP "
"error {} for URL {}".format(req, exc, link)
f"Could not install requirement {req} because of HTTP "
f"error {exc} for URL {link}"
)
else:
file_path = self._downloaded[link.url]
@@ -561,12 +623,15 @@ class RequirementPreparer:
# Make sure we have a hash in download_info. If we got it as part of the
# URL, it will have been verified and we can rely on it. Otherwise we
# compute it from the downloaded file.
# FIXME: https://github.com/pypa/pip/issues/11943
if (
isinstance(req.download_info.info, ArchiveInfo)
and not req.download_info.info.hash
and not req.download_info.info.hashes
and local_file
):
hash = hash_file(local_file.path)[0].hexdigest()
# We populate info.hash for backward compatibility.
# This will automatically populate info.hashes.
req.download_info.info.hash = f"sha256={hash}"
# For use in later processing,
@@ -621,9 +686,9 @@ class RequirementPreparer:
with indent_log():
if self.require_hashes:
raise InstallationError(
"The editable requirement {} cannot be installed when "
f"The editable requirement {req} cannot be installed when "
"requiring hashes, because there is no single file to "
"hash.".format(req)
"hash."
)
req.ensure_has_source_dir(self.src_dir)
req.update_editable()
@@ -651,7 +716,7 @@ class RequirementPreparer:
assert req.satisfied_by, "req should have been satisfied but isn't"
assert skip_reason is not None, (
"did not get skip reason skipped but req.satisfied_by "
"is set to {}".format(req.satisfied_by)
f"is set to {req.satisfied_by}"
)
logger.info(
"Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version