nem sei pq tantos arquivos

This commit is contained in:
2025-02-11 11:07:58 -03:00
parent 66fb4eb17b
commit 2da09a8a25
1841 changed files with 115867 additions and 77478 deletions

View File

@@ -1,5 +1,6 @@
import collections
import logging
from dataclasses import dataclass
from typing import Generator, List, Optional, Sequence, Tuple
from pip._internal.utils.logging import indent_log
@@ -18,12 +19,9 @@ __all__ = [
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class InstallationResult:
def __init__(self, name: str) -> None:
self.name = name
def __repr__(self) -> str:
return f"InstallationResult(name={self.name!r})"
name: str
def _validate_requirements(
@@ -36,7 +34,6 @@ def _validate_requirements(
def install_given_reqs(
requirements: List[InstallRequirement],
install_options: List[str],
global_options: Sequence[str],
root: Optional[str],
home: Optional[str],
@@ -71,7 +68,6 @@ def install_given_reqs(
try:
requirement.install(
install_options,
global_options,
root=root,
home=home,

View File

@@ -8,10 +8,12 @@ These are meant to be used elsewhere within pip to create instances of
InstallRequirement.
"""
import copy
import logging
import os
import re
from typing import Any, Dict, Optional, Set, Tuple, Union
from dataclasses import dataclass
from typing import Collection, Dict, List, Optional, Set, Tuple, Union
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
@@ -57,6 +59,31 @@ def convert_extras(extras: Optional[str]) -> Set[str]:
return get_requirement("placeholder" + extras.lower()).extras
def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requirement:
"""
Returns a new requirement based on the given one, with the supplied extras. If the
given requirement already has extras those are replaced (or dropped if no new extras
are given).
"""
match: Optional[re.Match[str]] = re.fullmatch(
# see https://peps.python.org/pep-0508/#complete-grammar
r"([\w\t .-]+)(\[[^\]]*\])?(.*)",
str(req),
flags=re.ASCII,
)
# ireq.req is a valid requirement so the regex should always match
assert (
match is not None
), f"regex match on requirement {req} failed, this should never happen"
pre: Optional[str] = match.group(1)
post: Optional[str] = match.group(3)
assert (
pre is not None and post is not None
), f"regex group selection for requirement {req} failed, this should never happen"
extras: str = "[{}]".format(",".join(sorted(new_extras)) if new_extras else "")
return get_requirement(f"{pre}{extras}{post}")
def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
"""Parses an editable requirement into:
- a requirement name
@@ -106,8 +133,8 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
package_name = link.egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '{}', please specify one "
"with #egg=your_package_name".format(editable_req)
f"Could not detect requirement name for '{editable_req}', "
"please specify one with #egg=your_package_name"
)
return package_name, url, set()
@@ -136,7 +163,7 @@ def check_first_requirement_in_file(filename: str) -> None:
# If there is a line continuation, drop it, and append the next line.
if line.endswith("\\"):
line = line[:-2].strip() + next(lines, "")
Requirement(line)
get_requirement(line)
return
@@ -165,18 +192,12 @@ def deduce_helpful_msg(req: str) -> str:
return msg
@dataclass(frozen=True)
class RequirementParts:
def __init__(
self,
requirement: Optional[Requirement],
link: Optional[Link],
markers: Optional[Marker],
extras: Set[str],
):
self.requirement = requirement
self.link = link
self.markers = markers
self.extras = extras
requirement: Optional[Requirement]
link: Optional[Link]
markers: Optional[Marker]
extras: Set[str]
def parse_req_from_editable(editable_req: str) -> RequirementParts:
@@ -184,9 +205,9 @@ def parse_req_from_editable(editable_req: str) -> RequirementParts:
if name is not None:
try:
req: Optional[Requirement] = Requirement(name)
except InvalidRequirement:
raise InstallationError(f"Invalid requirement: '{name}'")
req: Optional[Requirement] = get_requirement(name)
except InvalidRequirement as exc:
raise InstallationError(f"Invalid requirement: {name!r}: {exc}")
else:
req = None
@@ -201,15 +222,16 @@ def parse_req_from_editable(editable_req: str) -> RequirementParts:
def install_req_from_editable(
editable_req: str,
comes_from: Optional[Union[InstallRequirement, str]] = None,
*,
use_pep517: Optional[bool] = None,
isolated: bool = False,
options: Optional[Dict[str, Any]] = None,
global_options: Optional[List[str]] = None,
hash_options: Optional[Dict[str, List[str]]] = None,
constraint: bool = False,
user_supplied: bool = False,
permit_editable_wheels: bool = False,
config_settings: Optional[Dict[str, str]] = None,
config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
) -> InstallRequirement:
parts = parse_req_from_editable(editable_req)
return InstallRequirement(
@@ -222,9 +244,8 @@ def install_req_from_editable(
constraint=constraint,
use_pep517=use_pep517,
isolated=isolated,
install_options=options.get("install_options", []) if options else [],
global_options=options.get("global_options", []) if options else [],
hash_options=options.get("hashes", {}) if options else {},
global_options=global_options,
hash_options=hash_options,
config_settings=config_settings,
extras=parts.extras,
)
@@ -338,8 +359,8 @@ def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementPar
def _parse_req_string(req_as_string: str) -> Requirement:
try:
req = get_requirement(req_as_string)
except InvalidRequirement:
return get_requirement(req_as_string)
except InvalidRequirement as exc:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req_as_string)
@@ -349,21 +370,10 @@ def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementPar
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = ""
msg = with_source(f"Invalid requirement: {req_as_string!r}")
msg = with_source(f"Invalid requirement: {req_as_string!r}: {exc}")
if add_msg:
msg += f"\nHint: {add_msg}"
raise InstallationError(msg)
else:
# Deprecate extras after specifiers: "name>=1.0[extras]"
# This currently works by accident because _strip_extras() parses
# any extras in the end of the string and those are saved in
# RequirementParts
for spec in req.specifier:
spec_str = str(spec)
if spec_str.endswith("]"):
msg = f"Extras after version '{spec_str}'."
raise InstallationError(msg)
return req
if req_as_string is not None:
req: Optional[Requirement] = _parse_req_string(req_as_string)
@@ -376,13 +386,15 @@ def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementPar
def install_req_from_line(
name: str,
comes_from: Optional[Union[str, InstallRequirement]] = None,
*,
use_pep517: Optional[bool] = None,
isolated: bool = False,
options: Optional[Dict[str, Any]] = None,
global_options: Optional[List[str]] = None,
hash_options: Optional[Dict[str, List[str]]] = None,
constraint: bool = False,
line_source: Optional[str] = None,
user_supplied: bool = False,
config_settings: Optional[Dict[str, str]] = None,
config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
) -> InstallRequirement:
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
@@ -399,9 +411,8 @@ def install_req_from_line(
markers=parts.markers,
use_pep517=use_pep517,
isolated=isolated,
install_options=options.get("install_options", []) if options else [],
global_options=options.get("global_options", []) if options else [],
hash_options=options.get("hashes", {}) if options else {},
global_options=global_options,
hash_options=hash_options,
config_settings=config_settings,
constraint=constraint,
extras=parts.extras,
@@ -415,12 +426,11 @@ def install_req_from_req_string(
isolated: bool = False,
use_pep517: Optional[bool] = None,
user_supplied: bool = False,
config_settings: Optional[Dict[str, str]] = None,
) -> InstallRequirement:
try:
req = get_requirement(req_string)
except InvalidRequirement:
raise InstallationError(f"Invalid requirement: '{req_string}'")
except InvalidRequirement as exc:
raise InstallationError(f"Invalid requirement: {req_string!r}: {exc}")
domains_not_allowed = [
PyPI.file_storage_domain,
@@ -436,7 +446,7 @@ def install_req_from_req_string(
raise InstallationError(
"Packages installed from PyPI cannot depend on packages "
"which are not also hosted on PyPI.\n"
"{} depends on {} ".format(comes_from.name, req)
f"{comes_from.name} depends on {req} "
)
return InstallRequirement(
@@ -445,7 +455,6 @@ def install_req_from_req_string(
isolated=isolated,
use_pep517=use_pep517,
user_supplied=user_supplied,
config_settings=config_settings,
)
@@ -454,7 +463,7 @@ def install_req_from_parsed_requirement(
isolated: bool = False,
use_pep517: Optional[bool] = None,
user_supplied: bool = False,
config_settings: Optional[Dict[str, str]] = None,
config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
) -> InstallRequirement:
if parsed_req.is_editable:
req = install_req_from_editable(
@@ -473,7 +482,14 @@ def install_req_from_parsed_requirement(
comes_from=parsed_req.comes_from,
use_pep517=use_pep517,
isolated=isolated,
options=parsed_req.options,
global_options=(
parsed_req.options.get("global_options", [])
if parsed_req.options
else []
),
hash_options=(
parsed_req.options.get("hashes", {}) if parsed_req.options else {}
),
constraint=parsed_req.constraint,
line_source=parsed_req.line_source,
user_supplied=user_supplied,
@@ -493,9 +509,52 @@ def install_req_from_link_and_ireq(
markers=ireq.markers,
use_pep517=ireq.use_pep517,
isolated=ireq.isolated,
install_options=ireq.install_options,
global_options=ireq.global_options,
hash_options=ireq.hash_options,
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
)
def install_req_drop_extras(ireq: InstallRequirement) -> InstallRequirement:
"""
Creates a new InstallationRequirement using the given template but without
any extras. Sets the original requirement as the new one's parent
(comes_from).
"""
return InstallRequirement(
req=(
_set_requirement_extras(ireq.req, set()) if ireq.req is not None else None
),
comes_from=ireq,
editable=ireq.editable,
link=ireq.link,
markers=ireq.markers,
use_pep517=ireq.use_pep517,
isolated=ireq.isolated,
global_options=ireq.global_options,
hash_options=ireq.hash_options,
constraint=ireq.constraint,
extras=[],
config_settings=ireq.config_settings,
user_supplied=ireq.user_supplied,
permit_editable_wheels=ireq.permit_editable_wheels,
)
def install_req_extend_extras(
ireq: InstallRequirement,
extras: Collection[str],
) -> InstallRequirement:
"""
Returns a copy of an installation requirement with some additional extras.
Makes a shallow copy of the ireq object.
"""
result = copy.copy(ireq)
result.extras = {*ireq.extras, *extras}
result.req = (
_set_requirement_extras(ireq.req, result.extras)
if ireq.req is not None
else None
)
return result

View File

@@ -2,11 +2,16 @@
Requirements file parsing
"""
import codecs
import locale
import logging
import optparse
import os
import re
import shlex
import sys
import urllib.parse
from dataclasses import dataclass
from optparse import Values
from typing import (
TYPE_CHECKING,
@@ -16,6 +21,7 @@ from typing import (
Generator,
Iterable,
List,
NoReturn,
Optional,
Tuple,
)
@@ -23,17 +29,10 @@ from typing import (
from pip._internal.cli import cmdoptions
from pip._internal.exceptions import InstallationError, RequirementsFileParseError
from pip._internal.models.search_scope import SearchScope
from pip._internal.network.session import PipSession
from pip._internal.network.utils import raise_for_status
from pip._internal.utils.encoding import auto_decode
from pip._internal.utils.urls import get_url_scheme
if TYPE_CHECKING:
# NoReturn introduced in 3.6.2; imported only for type checking to maintain
# pip compatibility with older patch versions of Python 3.6
from typing import NoReturn
from pip._internal.index.package_finder import PackageFinder
from pip._internal.network.session import PipSession
__all__ = ["parse_requirements"]
@@ -69,63 +68,87 @@ SUPPORTED_OPTIONS: List[Callable[..., optparse.Option]] = [
# options to be passed to requirements
SUPPORTED_OPTIONS_REQ: List[Callable[..., optparse.Option]] = [
cmdoptions.install_options,
cmdoptions.global_options,
cmdoptions.hash,
cmdoptions.config_settings,
]
SUPPORTED_OPTIONS_EDITABLE_REQ: List[Callable[..., optparse.Option]] = [
cmdoptions.config_settings,
]
# the 'dest' string values
SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
SUPPORTED_OPTIONS_EDITABLE_REQ_DEST = [
str(o().dest) for o in SUPPORTED_OPTIONS_EDITABLE_REQ
]
# order of BOMS is important: codecs.BOM_UTF16_LE is a prefix of codecs.BOM_UTF32_LE
# so data.startswith(BOM_UTF16_LE) would be true for UTF32_LE data
BOMS: List[Tuple[bytes, str]] = [
(codecs.BOM_UTF8, "utf-8"),
(codecs.BOM_UTF32, "utf-32"),
(codecs.BOM_UTF32_BE, "utf-32-be"),
(codecs.BOM_UTF32_LE, "utf-32-le"),
(codecs.BOM_UTF16, "utf-16"),
(codecs.BOM_UTF16_BE, "utf-16-be"),
(codecs.BOM_UTF16_LE, "utf-16-le"),
]
PEP263_ENCODING_RE = re.compile(rb"coding[:=]\s*([-\w.]+)")
DEFAULT_ENCODING = "utf-8"
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class ParsedRequirement:
def __init__(
self,
requirement: str,
is_editable: bool,
comes_from: str,
constraint: bool,
options: Optional[Dict[str, Any]] = None,
line_source: Optional[str] = None,
) -> None:
self.requirement = requirement
self.is_editable = is_editable
self.comes_from = comes_from
self.options = options
self.constraint = constraint
self.line_source = line_source
# TODO: replace this with slots=True when dropping Python 3.9 support.
__slots__ = (
"requirement",
"is_editable",
"comes_from",
"constraint",
"options",
"line_source",
)
requirement: str
is_editable: bool
comes_from: str
constraint: bool
options: Optional[Dict[str, Any]]
line_source: Optional[str]
@dataclass(frozen=True)
class ParsedLine:
def __init__(
self,
filename: str,
lineno: int,
args: str,
opts: Values,
constraint: bool,
) -> None:
self.filename = filename
self.lineno = lineno
self.opts = opts
self.constraint = constraint
__slots__ = ("filename", "lineno", "args", "opts", "constraint")
if args:
self.is_requirement = True
self.is_editable = False
self.requirement = args
elif opts.editables:
self.is_requirement = True
self.is_editable = True
filename: str
lineno: int
args: str
opts: Values
constraint: bool
@property
def is_editable(self) -> bool:
return bool(self.opts.editables)
@property
def requirement(self) -> Optional[str]:
if self.args:
return self.args
elif self.is_editable:
# We don't support multiple -e on one line
self.requirement = opts.editables[0]
else:
self.is_requirement = False
return self.opts.editables[0]
return None
def parse_requirements(
filename: str,
session: PipSession,
session: "PipSession",
finder: Optional["PackageFinder"] = None,
options: Optional[optparse.Values] = None,
constraint: bool = False,
@@ -166,7 +189,6 @@ def handle_requirement_line(
line: ParsedLine,
options: Optional[optparse.Values] = None,
) -> ParsedRequirement:
# preserve for the nested code path
line_comes_from = "{} {} (line {})".format(
"-c" if line.constraint else "-r",
@@ -174,33 +196,27 @@ def handle_requirement_line(
line.lineno,
)
assert line.is_requirement
assert line.requirement is not None
# get the options that apply to requirements
if line.is_editable:
# For editable requirements, we don't support per-requirement
# options, so just return the parsed requirement.
return ParsedRequirement(
requirement=line.requirement,
is_editable=line.is_editable,
comes_from=line_comes_from,
constraint=line.constraint,
)
supported_dest = SUPPORTED_OPTIONS_EDITABLE_REQ_DEST
else:
# get the options that apply to requirements
req_options = {}
for dest in SUPPORTED_OPTIONS_REQ_DEST:
if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
req_options[dest] = line.opts.__dict__[dest]
supported_dest = SUPPORTED_OPTIONS_REQ_DEST
req_options = {}
for dest in supported_dest:
if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
req_options[dest] = line.opts.__dict__[dest]
line_source = f"line {line.lineno} of {line.filename}"
return ParsedRequirement(
requirement=line.requirement,
is_editable=line.is_editable,
comes_from=line_comes_from,
constraint=line.constraint,
options=req_options,
line_source=line_source,
)
line_source = f"line {line.lineno} of {line.filename}"
return ParsedRequirement(
requirement=line.requirement,
is_editable=line.is_editable,
comes_from=line_comes_from,
constraint=line.constraint,
options=req_options,
line_source=line_source,
)
def handle_option_line(
@@ -209,8 +225,14 @@ def handle_option_line(
lineno: int,
finder: Optional["PackageFinder"] = None,
options: Optional[optparse.Values] = None,
session: Optional[PipSession] = None,
session: Optional["PipSession"] = None,
) -> None:
if opts.hashes:
logger.warning(
"%s line %s has --hash but no requirement, and will be ignored.",
filename,
lineno,
)
if options:
# percolate options upward
@@ -271,7 +293,7 @@ def handle_line(
line: ParsedLine,
options: Optional[optparse.Values] = None,
finder: Optional["PackageFinder"] = None,
session: Optional[PipSession] = None,
session: Optional["PipSession"] = None,
) -> Optional[ParsedRequirement]:
"""Handle a single parsed requirements line; This can result in
creating/yielding requirements, or updating the finder.
@@ -296,7 +318,7 @@ def handle_line(
affect the finder.
"""
if line.is_requirement:
if line.requirement is not None:
parsed_req = handle_requirement_line(line, options)
return parsed_req
else:
@@ -314,7 +336,7 @@ def handle_line(
class RequirementsFileParser:
def __init__(
self,
session: PipSession,
session: "PipSession",
line_parser: LineParser,
) -> None:
self._session = session
@@ -324,13 +346,18 @@ class RequirementsFileParser:
self, filename: str, constraint: bool
) -> Generator[ParsedLine, None, None]:
"""Parse a given file, yielding parsed lines."""
yield from self._parse_and_recurse(filename, constraint)
yield from self._parse_and_recurse(
filename, constraint, [{os.path.abspath(filename): None}]
)
def _parse_and_recurse(
self, filename: str, constraint: bool
self,
filename: str,
constraint: bool,
parsed_files_stack: List[Dict[str, Optional[str]]],
) -> Generator[ParsedLine, None, None]:
for line in self._parse_file(filename, constraint):
if not line.is_requirement and (
if line.requirement is None and (
line.opts.requirements or line.opts.constraints
):
# parse a nested requirements file
@@ -348,12 +375,30 @@ class RequirementsFileParser:
# original file and nested file are paths
elif not SCHEME_RE.search(req_path):
# do a join so relative paths work
req_path = os.path.join(
os.path.dirname(filename),
req_path,
# and then abspath so that we can identify recursive references
req_path = os.path.abspath(
os.path.join(
os.path.dirname(filename),
req_path,
)
)
yield from self._parse_and_recurse(req_path, nested_constraint)
parsed_files = parsed_files_stack[0]
if req_path in parsed_files:
initial_file = parsed_files[req_path]
tail = (
f" and again in {initial_file}"
if initial_file is not None
else ""
)
raise RequirementsFileParseError(
f"{req_path} recursively references itself in {filename}{tail}"
)
# Keeping a track where was each file first included in
new_parsed_files = parsed_files.copy()
new_parsed_files[req_path] = filename
yield from self._parse_and_recurse(
req_path, nested_constraint, [new_parsed_files, *parsed_files_stack]
)
else:
yield line
@@ -519,7 +564,7 @@ def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines:
yield line_number, line
def get_file_content(url: str, session: PipSession) -> Tuple[str, str]:
def get_file_content(url: str, session: "PipSession") -> Tuple[str, str]:
"""Gets the content of a file; it may be a filename, file: URL, or
http: URL. Returns (location, content). Content is unicode.
Respects # -*- coding: declarations on the retrieved files.
@@ -527,10 +572,12 @@ def get_file_content(url: str, session: PipSession) -> Tuple[str, str]:
:param url: File path or url.
:param session: PipSession instance.
"""
scheme = get_url_scheme(url)
scheme = urllib.parse.urlsplit(url).scheme
# Pip has special support for file:// URLs (LocalFSAdapter).
if scheme in ["http", "https", "file"]:
# Delay importing heavy network modules until absolutely necessary.
from pip._internal.network.utils import raise_for_status
resp = session.get(url)
raise_for_status(resp)
return resp.url, resp.text
@@ -538,7 +585,39 @@ def get_file_content(url: str, session: PipSession) -> Tuple[str, str]:
# Assume this is a bare path.
try:
with open(url, "rb") as f:
content = auto_decode(f.read())
raw_content = f.read()
except OSError as exc:
raise InstallationError(f"Could not open requirements file: {exc}")
content = _decode_req_file(raw_content, url)
return url, content
def _decode_req_file(data: bytes, url: str) -> str:
for bom, encoding in BOMS:
if data.startswith(bom):
return data[len(bom) :].decode(encoding)
for line in data.split(b"\n")[:2]:
if line[0:1] == b"#":
result = PEP263_ENCODING_RE.search(line)
if result is not None:
encoding = result.groups()[0].decode("ascii")
return data.decode(encoding)
try:
return data.decode(DEFAULT_ENCODING)
except UnicodeDecodeError:
locale_encoding = locale.getpreferredencoding(False) or sys.getdefaultencoding()
logging.warning(
"unable to decode data from %s with default encoding %s, "
"falling back to encoding from locale: %s. "
"If this is intentional you should specify the encoding with a "
"PEP-263 style comment, e.g. '# -*- coding: %s -*-'",
url,
DEFAULT_ENCODING,
locale_encoding,
locale_encoding,
)
return data.decode(locale_encoding)

View File

@@ -1,6 +1,3 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import functools
import logging
import os
@@ -8,8 +5,8 @@ import shutil
import sys
import uuid
import zipfile
from enum import Enum
from optparse import Values
from pathlib import Path
from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union
from pip._vendor.packaging.markers import Marker
@@ -21,7 +18,7 @@ from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment
from pip._internal.exceptions import InstallationError, LegacyInstallFailure
from pip._internal.exceptions import InstallationError, PreviousBuildDirError
from pip._internal.locations import get_scheme
from pip._internal.metadata import (
BaseDistribution,
@@ -40,15 +37,10 @@ from pip._internal.operations.build.metadata_legacy import (
from pip._internal.operations.install.editable_legacy import (
install_editable as install_editable_legacy,
)
from pip._internal.operations.install.legacy import install as install_legacy
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
from pip._internal.req.req_uninstall import UninstallPathSet
from pip._internal.utils.deprecation import LegacyInstallReason, deprecated
from pip._internal.utils.direct_url_helpers import (
direct_url_for_editable,
direct_url_from_link,
)
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.misc import (
ConfiguredBuildBackendHookCaller,
@@ -56,11 +48,14 @@ from pip._internal.utils.misc import (
backup_dir,
display_path,
hide_url,
is_installable_dir,
redact_auth_from_requirement,
redact_auth_from_url,
)
from pip._internal.utils.packaging import safe_extra
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.subprocess import runner_with_spinner_message
from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
from pip._internal.utils.unpacking import unpack_file
from pip._internal.utils.virtualenv import running_under_virtualenv
from pip._internal.vcs import vcs
@@ -83,10 +78,10 @@ class InstallRequirement:
markers: Optional[Marker] = None,
use_pep517: Optional[bool] = None,
isolated: bool = False,
install_options: Optional[List[str]] = None,
*,
global_options: Optional[List[str]] = None,
hash_options: Optional[Dict[str, List[str]]] = None,
config_settings: Optional[Dict[str, str]] = None,
config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
constraint: bool = False,
extras: Collection[str] = (),
user_supplied: bool = False,
@@ -98,7 +93,6 @@ class InstallRequirement:
self.constraint = constraint
self.editable = editable
self.permit_editable_wheels = permit_editable_wheels
self.legacy_install_reason: Optional[LegacyInstallReason] = None
# source_dir is the local directory where the linked requirement is
# located, or unpacked. In case unpacking is needed, creating and
@@ -111,11 +105,17 @@ class InstallRequirement:
if link.is_file:
self.source_dir = os.path.normpath(os.path.abspath(link.file_path))
# original_link is the direct URL that was provided by the user for the
# requirement, either directly or via a constraints file.
if link is None and req and req.url:
# PEP 508 URL requirement
link = Link(req.url)
self.link = self.original_link = link
self.original_link_is_in_wheel_cache = False
# When this InstallRequirement is a wheel obtained from the cache of locally
# built wheels, this is the source link corresponding to the cache entry, which
# was used to download and build the cached wheel.
self.cached_wheel_source_link: Optional[Link] = None
# Information about the location of the artifact that was downloaded . This
# property is guaranteed to be set in resolver results.
@@ -129,7 +129,7 @@ class InstallRequirement:
if extras:
self.extras = extras
elif req:
self.extras = {safe_extra(extra) for extra in req.extras}
self.extras = req.extras
else:
self.extras = set()
if markers is None and req:
@@ -146,7 +146,6 @@ class InstallRequirement:
# Set to True after successful installation
self.install_succeeded: Optional[bool] = None
# Supplied options
self.install_options = install_options if install_options else []
self.global_options = global_options if global_options else []
self.hash_options = hash_options if hash_options else {}
self.config_settings = config_settings
@@ -182,14 +181,27 @@ class InstallRequirement:
# but after loading this flag should be treated as read only.
self.use_pep517 = use_pep517
# If config settings are provided, enforce PEP 517.
if self.config_settings:
if self.use_pep517 is False:
logger.warning(
"--no-use-pep517 ignored for %s "
"because --config-settings are specified.",
self,
)
self.use_pep517 = True
# This requirement needs more preparation before it can be built
self.needs_more_preparation = False
# This requirement needs to be unpacked before it can be installed.
self._archive_source: Optional[Path] = None
def __str__(self) -> str:
if self.req:
s = str(self.req)
s = redact_auth_from_requirement(self.req)
if self.link:
s += " from {}".format(redact_auth_from_url(self.link.url))
s += f" from {redact_auth_from_url(self.link.url)}"
elif self.link:
s = redact_auth_from_url(self.link.url)
else:
@@ -210,8 +222,9 @@ class InstallRequirement:
return s
def __repr__(self) -> str:
return "<{} object: {} editable={!r}>".format(
self.__class__.__name__, str(self), self.editable
return (
f"<{self.__class__.__name__} object: "
f"{str(self)} editable={self.editable!r}>"
)
def format_debug(self) -> str:
@@ -219,7 +232,7 @@ class InstallRequirement:
attributes = vars(self)
names = sorted(attributes)
state = ("{}={!r}".format(attr, attributes[attr]) for attr in sorted(names))
state = (f"{attr}={attributes[attr]!r}" for attr in sorted(names))
return "<{name} object: {{{state}}}>".format(
name=self.__class__.__name__,
state=", ".join(state),
@@ -232,7 +245,7 @@ class InstallRequirement:
return None
return self.req.name
@functools.lru_cache() # use cached_property in python 3.8+
@functools.cached_property
def supports_pyproject_editable(self) -> bool:
if not self.use_pep517:
return False
@@ -246,15 +259,22 @@ class InstallRequirement:
@property
def specifier(self) -> SpecifierSet:
assert self.req is not None
return self.req.specifier
@property
def is_direct(self) -> bool:
"""Whether this requirement was specified as a direct URL."""
return self.original_link is not None
@property
def is_pinned(self) -> bool:
"""Return whether I am pinned to an exact version.
For example, some-package==1.2 is pinned; some-package>1.2 is not.
"""
specifiers = self.specifier
assert self.req is not None
specifiers = self.req.specifier
return len(specifiers) == 1 and next(iter(specifiers)).operator in {"==", "==="}
def match_markers(self, extras_requested: Optional[Iterable[str]] = None) -> bool:
@@ -295,8 +315,14 @@ class InstallRequirement:
"""
good_hashes = self.hash_options.copy()
link = self.link if trust_internet else self.original_link
if trust_internet:
link = self.link
elif self.is_direct and self.user_supplied:
link = self.original_link
else:
link = None
if link and link.hash:
assert link.hash_name is not None
good_hashes.setdefault(link.hash_name, []).append(link.hash)
return Hashes(good_hashes)
@@ -306,6 +332,7 @@ class InstallRequirement:
return None
s = str(self.req)
if self.comes_from:
comes_from: Optional[str]
if isinstance(self.comes_from, str):
comes_from = self.comes_from
else:
@@ -337,7 +364,7 @@ class InstallRequirement:
# When parallel builds are enabled, add a UUID to the build directory
# name so multiple builds do not interfere with each other.
dir_name: str = canonicalize_name(self.name)
dir_name: str = canonicalize_name(self.req.name)
if parallel_builds:
dir_name = f"{dir_name}_{uuid.uuid4().hex}"
@@ -369,7 +396,7 @@ class InstallRequirement:
else:
op = "==="
self.req = Requirement(
self.req = get_requirement(
"".join(
[
self.metadata["Name"],
@@ -380,6 +407,7 @@ class InstallRequirement:
)
def warn_on_mismatching_name(self) -> None:
assert self.req is not None
metadata_name = canonicalize_name(self.metadata["Name"])
if canonicalize_name(self.req.name) == metadata_name:
# Everything is fine.
@@ -394,7 +422,7 @@ class InstallRequirement:
metadata_name,
self.name,
)
self.req = Requirement(metadata_name)
self.req = get_requirement(metadata_name)
def check_if_exists(self, use_user_site: bool) -> None:
"""Find an installed distribution that satisfies or conflicts
@@ -440,9 +468,16 @@ class InstallRequirement:
return False
return self.link.is_wheel
@property
def is_wheel_from_cache(self) -> bool:
# When True, it means that this InstallRequirement is a local wheel file in the
# cache of locally built wheels.
return self.cached_wheel_source_link is not None
# Things valid for sdists
@property
def unpacked_source_directory(self) -> str:
assert self.source_dir, f"No source dir for {self}"
return os.path.join(
self.source_dir, self.link and self.link.subdirectory_fragment or ""
)
@@ -479,6 +514,7 @@ class InstallRequirement:
)
if pyproject_toml_data is None:
assert not self.config_settings
self.use_pep517 = False
return
@@ -502,7 +538,7 @@ class InstallRequirement:
if (
self.editable
and self.use_pep517
and not self.supports_pyproject_editable()
and not self.supports_pyproject_editable
and not os.path.isfile(self.setup_py_path)
and not os.path.isfile(self.setup_cfg_path)
):
@@ -520,7 +556,7 @@ class InstallRequirement:
Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
assert self.source_dir, f"No source dir for {self}"
details = self.name or f"from {self.link}"
if self.use_pep517:
@@ -528,7 +564,7 @@ class InstallRequirement:
if (
self.editable
and self.permit_editable_wheels
and self.supports_pyproject_editable()
and self.supports_pyproject_editable
):
self.metadata_directory = generate_editable_metadata(
build_env=self.build_env,
@@ -569,8 +605,10 @@ class InstallRequirement:
if self.metadata_directory:
return get_directory_distribution(self.metadata_directory)
elif self.local_file_path and self.is_wheel:
assert self.req is not None
return get_wheel_distribution(
FilesystemWheel(self.local_file_path), canonicalize_name(self.name)
FilesystemWheel(self.local_file_path),
canonicalize_name(self.req.name),
)
raise AssertionError(
f"InstallRequirement {self} has no metadata directory and no wheel: "
@@ -578,9 +616,9 @@ class InstallRequirement:
)
def assert_source_matches_version(self) -> None:
assert self.source_dir
assert self.source_dir, f"No source dir for {self}"
version = self.metadata["version"]
if self.req.specifier and version not in self.req.specifier:
if self.req and self.req.specifier and version not in self.req.specifier:
logger.warning(
"Requested %s, but installing version %s",
self,
@@ -617,6 +655,27 @@ class InstallRequirement:
parallel_builds=parallel_builds,
)
def needs_unpacked_archive(self, archive_source: Path) -> None:
assert self._archive_source is None
self._archive_source = archive_source
def ensure_pristine_source_checkout(self) -> None:
"""Ensure the source directory has not yet been built in."""
assert self.source_dir is not None
if self._archive_source is not None:
unpack_file(str(self._archive_source), self.source_dir)
elif is_installable_dir(self.source_dir):
# If a checkout exists, it's unwise to keep going.
# version inconsistencies are logged later, but do not fail
# the installation.
raise PreviousBuildDirError(
f"pip can't proceed with requirements '{self}' due to a "
f"pre-existing build directory ({self.source_dir}). This is likely "
"due to a previous installation that failed . pip is "
"being responsible and not assuming it can delete this. "
"Please delete it and try again."
)
# For editable installations
def update_editable(self) -> None:
if not self.link:
@@ -673,9 +732,10 @@ class InstallRequirement:
name = name.replace(os.path.sep, "/")
return name
assert self.req is not None
path = os.path.join(parentdir, path)
name = _clean_zip_name(path, rootdir)
return self.name + "/" + name
return self.req.name + "/" + name
def archive(self, build_dir: Optional[str]) -> None:
"""Saves archive to provided build_dir.
@@ -692,8 +752,8 @@ class InstallRequirement:
if os.path.exists(archive_path):
response = ask_path_exists(
"The file {} exists. (i)gnore, (w)ipe, "
"(b)ackup, (a)bort ".format(display_path(archive_path)),
f"The file {display_path(archive_path)} exists. (i)gnore, (w)ipe, "
"(b)ackup, (a)bort ",
("i", "w", "b", "a"),
)
if response == "i":
@@ -746,7 +806,6 @@ class InstallRequirement:
def install(
self,
install_options: List[str],
global_options: Optional[Sequence[str]] = None,
root: Optional[str] = None,
home: Optional[str] = None,
@@ -755,8 +814,9 @@ class InstallRequirement:
use_user_site: bool = False,
pycompile: bool = True,
) -> None:
assert self.req is not None
scheme = get_scheme(
self.name,
self.req.name,
user=use_user_site,
home=home,
root=root,
@@ -764,99 +824,60 @@ class InstallRequirement:
prefix=prefix,
)
global_options = global_options if global_options is not None else []
if self.editable and not self.is_wheel:
install_editable_legacy(
install_options,
global_options,
prefix=prefix,
home=home,
use_user_site=use_user_site,
name=self.name,
setup_py_path=self.setup_py_path,
isolated=self.isolated,
build_env=self.build_env,
unpacked_source_directory=self.unpacked_source_directory,
deprecated(
reason=(
f"Legacy editable install of {self} (setup.py develop) "
"is deprecated."
),
replacement=(
"to add a pyproject.toml or enable --use-pep517, "
"and use setuptools >= 64. "
"If the resulting installation is not behaving as expected, "
"try using --config-settings editable_mode=compat. "
"Please consult the setuptools documentation for more information"
),
gone_in="25.1",
issue=11457,
)
self.install_succeeded = True
return
if self.is_wheel:
assert self.local_file_path
direct_url = None
# TODO this can be refactored to direct_url = self.download_info
if self.editable:
direct_url = direct_url_for_editable(self.unpacked_source_directory)
elif self.original_link:
direct_url = direct_url_from_link(
self.original_link,
self.source_dir,
self.original_link_is_in_wheel_cache,
if self.config_settings:
logger.warning(
"--config-settings ignored for legacy editable install of %s. "
"Consider upgrading to a version of setuptools "
"that supports PEP 660 (>= 64).",
self,
)
install_wheel(
self.name,
self.local_file_path,
scheme=scheme,
req_description=str(self.req),
pycompile=pycompile,
warn_script_location=warn_script_location,
direct_url=direct_url,
requested=self.user_supplied,
install_editable_legacy(
global_options=global_options if global_options is not None else [],
prefix=prefix,
home=home,
use_user_site=use_user_site,
name=self.req.name,
setup_py_path=self.setup_py_path,
isolated=self.isolated,
build_env=self.build_env,
unpacked_source_directory=self.unpacked_source_directory,
)
self.install_succeeded = True
return
# TODO: Why don't we do this for editable installs?
assert self.is_wheel
assert self.local_file_path
# Extend the list of global and install options passed on to
# the setup.py call with the ones from the requirements file.
# Options specified in requirements file override those
# specified on the command line, since the last option given
# to setup.py is the one that is used.
global_options = list(global_options) + self.global_options
install_options = list(install_options) + self.install_options
try:
if (
self.legacy_install_reason is not None
and self.legacy_install_reason.emit_before_install
):
self.legacy_install_reason.emit_deprecation(self.name)
success = install_legacy(
install_options=install_options,
global_options=global_options,
root=root,
home=home,
prefix=prefix,
use_user_site=use_user_site,
pycompile=pycompile,
scheme=scheme,
setup_py_path=self.setup_py_path,
isolated=self.isolated,
req_name=self.name,
build_env=self.build_env,
unpacked_source_directory=self.unpacked_source_directory,
req_description=str(self.req),
)
except LegacyInstallFailure as exc:
self.install_succeeded = False
raise exc
except Exception:
self.install_succeeded = True
raise
self.install_succeeded = success
if (
success
and self.legacy_install_reason is not None
and self.legacy_install_reason.emit_after_success
):
self.legacy_install_reason.emit_deprecation(self.name)
install_wheel(
self.req.name,
self.local_file_path,
scheme=scheme,
req_description=str(self.req),
pycompile=pycompile,
warn_script_location=warn_script_location,
direct_url=self.download_info if self.is_direct else None,
requested=self.user_supplied,
)
self.install_succeeded = True
def check_invalid_constraint_type(req: InstallRequirement) -> str:
# Check for unsupported forms
problem = ""
if not req.name:
@@ -893,54 +914,21 @@ def _has_option(options: Values, reqs: List[InstallRequirement], option: str) ->
return False
def _install_option_ignored(
install_options: List[str], reqs: List[InstallRequirement]
) -> bool:
for req in reqs:
if (install_options or req.install_options) and not req.use_pep517:
return False
return True
class LegacySetupPyOptionsCheckMode(Enum):
INSTALL = 1
WHEEL = 2
DOWNLOAD = 3
def check_legacy_setup_py_options(
options: Values,
reqs: List[InstallRequirement],
mode: LegacySetupPyOptionsCheckMode,
) -> None:
has_install_options = _has_option(options, reqs, "install_options")
has_build_options = _has_option(options, reqs, "build_options")
has_global_options = _has_option(options, reqs, "global_options")
legacy_setup_py_options_present = (
has_install_options or has_build_options or has_global_options
)
if not legacy_setup_py_options_present:
return
options.format_control.disallow_binaries()
logger.warning(
"Implying --no-binary=:all: due to the presence of "
"--build-option / --global-option / --install-option. "
"Consider using --config-settings for more flexibility.",
)
if mode == LegacySetupPyOptionsCheckMode.INSTALL and has_install_options:
if _install_option_ignored(options.install_options, reqs):
logger.warning(
"Ignoring --install-option when building using PEP 517",
)
else:
deprecated(
reason=(
"--install-option is deprecated because "
"it forces pip to use the 'setup.py install' "
"command which is itself deprecated."
),
issue=11358,
replacement="to use --config-settings",
gone_in="23.1",
)
if has_build_options or has_global_options:
deprecated(
reason="--build-option and --global-option are deprecated.",
issue=11859,
replacement="to use --config-settings",
gone_in=None,
)
logger.warning(
"Implying --no-binary=:all: due to the presence of "
"--build-option / --global-option. "
)
options.format_control.disallow_binaries()

View File

@@ -5,14 +5,15 @@ import sysconfig
from importlib.util import cache_from_source
from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Set, Tuple
from pip._internal.exceptions import UninstallationError
from pip._internal.exceptions import LegacyDistutilsInstall, UninstallMissingRecord
from pip._internal.locations import get_bin_prefix, get_bin_user
from pip._internal.metadata import BaseDistribution
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.egg_link import egg_link_path_from_location
from pip._internal.utils.logging import getLogger, indent_log
from pip._internal.utils.misc import ask, is_local, normalize_path, renames, rmtree
from pip._internal.utils.misc import ask, normalize_path, renames, rmtree
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
from pip._internal.utils.virtualenv import running_under_virtualenv
logger = getLogger(__name__)
@@ -60,7 +61,7 @@ def uninstallation_paths(dist: BaseDistribution) -> Generator[str, None, None]:
UninstallPathSet.add() takes care of the __pycache__ .py[co].
If RECORD is not found, raises UninstallationError,
If RECORD is not found, raises an error,
with possible information from the INSTALLER file.
https://packaging.python.org/specifications/recording-installed-packages/
@@ -70,17 +71,7 @@ def uninstallation_paths(dist: BaseDistribution) -> Generator[str, None, None]:
entries = dist.iter_declared_entries()
if entries is None:
msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist)
installer = dist.installer
if not installer or installer == "pip":
dep = "{}=={}".format(dist.raw_name, dist.version)
msg += (
" You might be able to recover from this via: "
"'pip install --force-reinstall --no-deps {}'.".format(dep)
)
else:
msg += " Hint: The package was installed by {}.".format(installer)
raise UninstallationError(msg)
raise UninstallMissingRecord(distribution=dist)
for entry in entries:
path = os.path.join(location, entry)
@@ -171,8 +162,7 @@ def compress_for_output_listing(paths: Iterable[str]) -> Tuple[Set[str], Set[str
folders.add(os.path.dirname(path))
files.add(path)
# probably this one https://github.com/python/mypy/issues/390
_normcased_files = set(map(os.path.normcase, files)) # type: ignore
_normcased_files = set(map(os.path.normcase, files))
folders = compact(folders)
@@ -273,7 +263,7 @@ class StashedUninstallPathSet:
def commit(self) -> None:
"""Commits the uninstall by removing stashed files."""
for _, save_dir in self._save_dirs.items():
for save_dir in self._save_dirs.values():
save_dir.cleanup()
self._moves = []
self._save_dirs = {}
@@ -312,6 +302,10 @@ class UninstallPathSet:
self._pth: Dict[str, UninstallPthEntries] = {}
self._dist = dist
self._moved_paths = StashedUninstallPathSet()
# Create local cache of normalize_path results. Creating an UninstallPathSet
# can result in hundreds/thousands of redundant calls to normalize_path with
# the same args, which hurts performance.
self._normalize_path_cached = functools.lru_cache(normalize_path)
def _permitted(self, path: str) -> bool:
"""
@@ -319,14 +313,17 @@ class UninstallPathSet:
remove/modify, False otherwise.
"""
return is_local(path)
# aka is_local, but caching normalized sys.prefix
if not running_under_virtualenv():
return True
return path.startswith(self._normalize_path_cached(sys.prefix))
def add(self, path: str) -> None:
head, tail = os.path.split(path)
# we normalize the head to resolve parent directory symlinks, but not
# the tail, since we only want to uninstall symlinks, not their targets
path = os.path.join(normalize_path(head), os.path.normcase(tail))
path = os.path.join(self._normalize_path_cached(head), os.path.normcase(tail))
if not os.path.exists(path):
return
@@ -341,7 +338,7 @@ class UninstallPathSet:
self.add(cache_from_source(path))
def add_pth(self, pth_file: str, entry: str) -> None:
pth_file = normalize_path(pth_file)
pth_file = self._normalize_path_cached(pth_file)
if self._permitted(pth_file):
if pth_file not in self._pth:
self._pth[pth_file] = UninstallPthEntries(pth_file)
@@ -360,7 +357,7 @@ class UninstallPathSet:
)
return
dist_name_version = f"{self._dist.raw_name}-{self._dist.version}"
dist_name_version = f"{self._dist.raw_name}-{self._dist.raw_version}"
logger.info("Uninstalling %s:", dist_name_version)
with indent_log():
@@ -502,13 +499,7 @@ class UninstallPathSet:
paths_to_remove.add(f"{path}.pyo")
elif dist.installed_by_distutils:
raise UninstallationError(
"Cannot uninstall {!r}. It is a distutils installed project "
"and thus we cannot accurately determine which files belong "
"to it which would lead to only a partial uninstall.".format(
dist.raw_name,
)
)
raise LegacyDistutilsInstall(distribution=dist)
elif dist.installed_as_egg:
# package installed by easy_install
@@ -531,12 +522,14 @@ class UninstallPathSet:
# above, so this only covers the setuptools-style editable.
with open(develop_egg_link) as fh:
link_pointer = os.path.normcase(fh.readline().strip())
normalized_link_pointer = normalize_path(link_pointer)
normalized_link_pointer = paths_to_remove._normalize_path_cached(
link_pointer
)
assert os.path.samefile(
normalized_link_pointer, normalized_dist_location
), (
f"Egg-link {link_pointer} does not match installed location of "
f"{dist.raw_name} (at {dist_location})"
f"Egg-link {develop_egg_link} (to {link_pointer}) does not match "
f"installed location of {dist.raw_name} (at {dist_location})"
)
paths_to_remove.add(develop_egg_link)
easy_install_pth = os.path.join(