Загрузить файлы в «venv/Lib/site-packages/pip/_internal/network»
This commit is contained in:
1
venv/Lib/site-packages/pip/_internal/network/__init__.py
Normal file
1
venv/Lib/site-packages/pip/_internal/network/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Contains purely network-related utilities."""
|
||||
568
venv/Lib/site-packages/pip/_internal/network/auth.py
Normal file
568
venv/Lib/site-packages/pip/_internal/network/auth.py
Normal file
@@ -0,0 +1,568 @@
|
||||
"""Network Authentication Helpers
|
||||
|
||||
Contains interface (MultiDomainBasicAuth) and associated glue code for
|
||||
providing credentials in the context of network requests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sysconfig
|
||||
import typing
|
||||
import urllib.parse
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import cache
|
||||
from os.path import commonprefix
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
|
||||
from pip._vendor.requests.utils import get_netrc_auth
|
||||
|
||||
from pip._internal.utils.logging import getLogger
|
||||
from pip._internal.utils.misc import (
|
||||
ask,
|
||||
ask_input,
|
||||
ask_password,
|
||||
remove_auth_from_url,
|
||||
split_auth_netloc_from_url,
|
||||
)
|
||||
from pip._internal.vcs.versioncontrol import AuthInfo
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from pip._vendor.requests import PreparedRequest
|
||||
from pip._vendor.requests.models import Response
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
KEYRING_DISABLED = False
|
||||
|
||||
|
||||
class Credentials(NamedTuple):
|
||||
url: str
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class KeyRingBaseProvider(ABC):
|
||||
"""Keyring base provider interface"""
|
||||
|
||||
has_keyring: bool
|
||||
|
||||
@abstractmethod
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None: ...
|
||||
|
||||
@abstractmethod
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None: ...
|
||||
|
||||
|
||||
class KeyRingNullProvider(KeyRingBaseProvider):
|
||||
"""Keyring null provider"""
|
||||
|
||||
has_keyring = False
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
return None
|
||||
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class KeyRingPythonProvider(KeyRingBaseProvider):
|
||||
"""Keyring interface which uses locally imported `keyring`"""
|
||||
|
||||
has_keyring = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
import keyring
|
||||
|
||||
self.keyring = keyring
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
# Support keyring's get_credential interface which supports getting
|
||||
# credentials without a username. This is only available for
|
||||
# keyring>=15.2.0.
|
||||
if hasattr(self.keyring, "get_credential"):
|
||||
logger.debug("Getting credentials from keyring for %s", url)
|
||||
cred = self.keyring.get_credential(url, username)
|
||||
if cred is not None:
|
||||
return cred.username, cred.password
|
||||
return None
|
||||
|
||||
if username is not None:
|
||||
logger.debug("Getting password from keyring for %s", url)
|
||||
password = self.keyring.get_password(url, username)
|
||||
if password:
|
||||
return username, password
|
||||
return None
|
||||
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
self.keyring.set_password(url, username, password)
|
||||
|
||||
|
||||
class KeyRingCliProvider(KeyRingBaseProvider):
|
||||
"""Provider which uses `keyring` cli
|
||||
|
||||
Instead of calling the keyring package installed alongside pip
|
||||
we call keyring on the command line which will enable pip to
|
||||
use which ever installation of keyring is available first in
|
||||
PATH.
|
||||
"""
|
||||
|
||||
has_keyring = True
|
||||
|
||||
def __init__(self, cmd: str) -> None:
|
||||
self.keyring = cmd
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
# This is the default implementation of keyring.get_credential
|
||||
# https://github.com/jaraco/keyring/blob/97689324abcf01bd1793d49063e7ca01e03d7d07/keyring/backend.py#L134-L139
|
||||
if username is not None:
|
||||
password = self._get_password(url, username)
|
||||
if password is not None:
|
||||
return username, password
|
||||
return None
|
||||
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
return self._set_password(url, username, password)
|
||||
|
||||
def _get_password(self, service_name: str, username: str) -> str | None:
|
||||
"""Mirror the implementation of keyring.get_password using cli"""
|
||||
if self.keyring is None:
|
||||
return None
|
||||
|
||||
cmd = [self.keyring, "get", service_name, username]
|
||||
env = os.environ.copy()
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
res = subprocess.run(
|
||||
cmd,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
if res.returncode:
|
||||
return None
|
||||
return res.stdout.decode("utf-8").strip(os.linesep)
|
||||
|
||||
def _set_password(self, service_name: str, username: str, password: str) -> None:
|
||||
"""Mirror the implementation of keyring.set_password using cli"""
|
||||
if self.keyring is None:
|
||||
return None
|
||||
env = os.environ.copy()
|
||||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
subprocess.run(
|
||||
[self.keyring, "set", service_name, username],
|
||||
input=f"{password}{os.linesep}".encode(),
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@cache
|
||||
def get_keyring_provider(provider: str) -> KeyRingBaseProvider:
|
||||
logger.verbose("Keyring provider requested: %s", provider)
|
||||
|
||||
# keyring has previously failed and been disabled
|
||||
if KEYRING_DISABLED:
|
||||
provider = "disabled"
|
||||
if provider in ["import", "auto"]:
|
||||
try:
|
||||
impl = KeyRingPythonProvider()
|
||||
logger.verbose("Keyring provider set: import")
|
||||
return impl
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
# In the event of an unexpected exception
|
||||
# we should warn the user
|
||||
msg = "Installed copy of keyring fails with exception %s"
|
||||
if provider == "auto":
|
||||
msg = msg + ", trying to find a keyring executable as a fallback"
|
||||
logger.warning(msg, exc, exc_info=logger.isEnabledFor(logging.DEBUG))
|
||||
if provider in ["subprocess", "auto"]:
|
||||
cli = shutil.which("keyring")
|
||||
if cli and cli.startswith(sysconfig.get_path("scripts")):
|
||||
# all code within this function is stolen from shutil.which implementation
|
||||
@typing.no_type_check
|
||||
def PATH_as_shutil_which_determines_it() -> str:
|
||||
path = os.environ.get("PATH", None)
|
||||
if path is None:
|
||||
try:
|
||||
path = os.confstr("CS_PATH")
|
||||
except (AttributeError, ValueError):
|
||||
# os.confstr() or CS_PATH is not available
|
||||
path = os.defpath
|
||||
# bpo-35755: Don't use os.defpath if the PATH environment variable is
|
||||
# set to an empty string
|
||||
|
||||
return path
|
||||
|
||||
scripts = Path(sysconfig.get_path("scripts"))
|
||||
|
||||
paths = []
|
||||
for path in PATH_as_shutil_which_determines_it().split(os.pathsep):
|
||||
p = Path(path)
|
||||
try:
|
||||
if not p.samefile(scripts):
|
||||
paths.append(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
path = os.pathsep.join(paths)
|
||||
|
||||
cli = shutil.which("keyring", path=path)
|
||||
|
||||
if cli:
|
||||
logger.verbose("Keyring provider set: subprocess with executable %s", cli)
|
||||
return KeyRingCliProvider(cli)
|
||||
|
||||
logger.verbose("Keyring provider set: disabled")
|
||||
return KeyRingNullProvider()
|
||||
|
||||
|
||||
class MultiDomainBasicAuth(AuthBase):
|
||||
def __init__(
|
||||
self,
|
||||
prompting: bool = True,
|
||||
index_urls: list[str] | None = None,
|
||||
keyring_provider: str = "auto",
|
||||
) -> None:
|
||||
self.prompting = prompting
|
||||
self.index_urls = index_urls
|
||||
self.keyring_provider = keyring_provider
|
||||
self.passwords: dict[str, AuthInfo] = {}
|
||||
# When the user is prompted to enter credentials and keyring is
|
||||
# available, we will offer to save them. If the user accepts,
|
||||
# this value is set to the credentials they entered. After the
|
||||
# request authenticates, the caller should call
|
||||
# ``save_credentials`` to save these.
|
||||
self._credentials_to_save: Credentials | None = None
|
||||
|
||||
@property
|
||||
def keyring_provider(self) -> KeyRingBaseProvider:
|
||||
return get_keyring_provider(self._keyring_provider)
|
||||
|
||||
@keyring_provider.setter
|
||||
def keyring_provider(self, provider: str) -> None:
|
||||
# The free function get_keyring_provider has been decorated with
|
||||
# functools.cache. If an exception occurs in get_keyring_auth that
|
||||
# cache will be cleared and keyring disabled, take that into account
|
||||
# if you want to remove this indirection.
|
||||
self._keyring_provider = provider
|
||||
|
||||
@property
|
||||
def use_keyring(self) -> bool:
|
||||
# We won't use keyring when --no-input is passed unless
|
||||
# a specific provider is requested because it might require
|
||||
# user interaction
|
||||
return self.prompting or self._keyring_provider not in ["auto", "disabled"]
|
||||
|
||||
def _get_keyring_auth(
|
||||
self,
|
||||
url: str | None,
|
||||
username: str | None,
|
||||
) -> AuthInfo | None:
|
||||
"""Return the tuple auth for a given url from keyring."""
|
||||
# Do nothing if no url was provided
|
||||
if not url:
|
||||
return None
|
||||
|
||||
try:
|
||||
return self.keyring_provider.get_auth_info(url, username)
|
||||
except Exception as exc:
|
||||
# Log the full exception (with stacktrace) at debug, so it'll only
|
||||
# show up when running in verbose mode.
|
||||
logger.debug("Keyring is skipped due to an exception", exc_info=True)
|
||||
# Always log a shortened version of the exception.
|
||||
logger.warning(
|
||||
"Keyring is skipped due to an exception: %s",
|
||||
str(exc),
|
||||
)
|
||||
global KEYRING_DISABLED
|
||||
KEYRING_DISABLED = True
|
||||
get_keyring_provider.cache_clear()
|
||||
return None
|
||||
|
||||
def _get_index_url(self, url: str) -> str | None:
|
||||
"""Return the original index URL matching the requested URL.
|
||||
|
||||
Cached or dynamically generated credentials may work against
|
||||
the original index URL rather than just the netloc.
|
||||
|
||||
The provided url should have had its username and password
|
||||
removed already. If the original index url had credentials then
|
||||
they will be included in the return value.
|
||||
|
||||
Returns None if no matching index was found, or if --no-index
|
||||
was specified by the user.
|
||||
"""
|
||||
if not url or not self.index_urls:
|
||||
return None
|
||||
|
||||
url = remove_auth_from_url(url).rstrip("/") + "/"
|
||||
parsed_url = urllib.parse.urlsplit(url)
|
||||
|
||||
candidates = []
|
||||
|
||||
for index in self.index_urls:
|
||||
index = index.rstrip("/") + "/"
|
||||
parsed_index = urllib.parse.urlsplit(remove_auth_from_url(index))
|
||||
if parsed_url == parsed_index:
|
||||
return index
|
||||
|
||||
if parsed_url.netloc != parsed_index.netloc:
|
||||
continue
|
||||
|
||||
candidate = urllib.parse.urlsplit(index)
|
||||
candidates.append(candidate)
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
candidates.sort(
|
||||
reverse=True,
|
||||
key=lambda candidate: commonprefix(
|
||||
[
|
||||
parsed_url.path,
|
||||
candidate.path,
|
||||
]
|
||||
).rfind("/"),
|
||||
)
|
||||
|
||||
return urllib.parse.urlunsplit(candidates[0])
|
||||
|
||||
def _get_new_credentials(
|
||||
self,
|
||||
original_url: str,
|
||||
*,
|
||||
allow_netrc: bool = True,
|
||||
allow_keyring: bool = False,
|
||||
) -> AuthInfo:
|
||||
"""Find and return credentials for the specified URL."""
|
||||
# Split the credentials and netloc from the url.
|
||||
url, netloc, url_user_password = split_auth_netloc_from_url(
|
||||
original_url,
|
||||
)
|
||||
|
||||
# Start with the credentials embedded in the url
|
||||
username, password = url_user_password
|
||||
if username is not None and password is not None:
|
||||
logger.debug("Found credentials in url for %s", netloc)
|
||||
return url_user_password
|
||||
|
||||
# Find a matching index url for this request
|
||||
index_url = self._get_index_url(url)
|
||||
if index_url:
|
||||
# Split the credentials from the url.
|
||||
index_info = split_auth_netloc_from_url(index_url)
|
||||
if index_info:
|
||||
index_url, _, index_url_user_password = index_info
|
||||
logger.debug("Found index url %s", index_url)
|
||||
|
||||
# If an index URL was found, try its embedded credentials
|
||||
if index_url and index_url_user_password[0] is not None:
|
||||
username, password = index_url_user_password
|
||||
if username is not None and password is not None:
|
||||
logger.debug("Found credentials in index url for %s", netloc)
|
||||
return index_url_user_password
|
||||
|
||||
# Get creds from netrc if we still don't have them
|
||||
if allow_netrc:
|
||||
netrc_auth = get_netrc_auth(original_url)
|
||||
if netrc_auth:
|
||||
logger.debug("Found credentials in netrc for %s", netloc)
|
||||
return netrc_auth
|
||||
|
||||
# If we don't have a password and keyring is available, use it.
|
||||
if allow_keyring:
|
||||
# The index url is more specific than the netloc, so try it first
|
||||
# fmt: off
|
||||
kr_auth = (
|
||||
self._get_keyring_auth(index_url, username) or
|
||||
self._get_keyring_auth(netloc, username)
|
||||
)
|
||||
# fmt: on
|
||||
if kr_auth:
|
||||
logger.debug("Found credentials in keyring for %s", netloc)
|
||||
return kr_auth
|
||||
|
||||
return username, password
|
||||
|
||||
def _get_url_and_credentials(
|
||||
self, original_url: str
|
||||
) -> tuple[str, str | None, str | None]:
|
||||
"""Return the credentials to use for the provided URL.
|
||||
|
||||
If allowed, netrc and keyring may be used to obtain the
|
||||
correct credentials.
|
||||
|
||||
Returns (url_without_credentials, username, password). Note
|
||||
that even if the original URL contains credentials, this
|
||||
function may return a different username and password.
|
||||
"""
|
||||
url, netloc, _ = split_auth_netloc_from_url(original_url)
|
||||
|
||||
# Try to get credentials from original url
|
||||
username, password = self._get_new_credentials(original_url)
|
||||
|
||||
# If credentials not found, use any stored credentials for this netloc.
|
||||
# Do this if either the username or the password is missing.
|
||||
# This accounts for the situation in which the user has specified
|
||||
# the username in the index url, but the password comes from keyring.
|
||||
if (username is None or password is None) and netloc in self.passwords:
|
||||
un, pw = self.passwords[netloc]
|
||||
# It is possible that the cached credentials are for a different username,
|
||||
# in which case the cache should be ignored.
|
||||
if username is None or username == un:
|
||||
username, password = un, pw
|
||||
|
||||
if username is not None or password is not None:
|
||||
# Convert the username and password if they're None, so that
|
||||
# this netloc will show up as "cached" in the conditional above.
|
||||
# Further, HTTPBasicAuth doesn't accept None, so it makes sense to
|
||||
# cache the value that is going to be used.
|
||||
username = username or ""
|
||||
password = password or ""
|
||||
|
||||
# Store any acquired credentials.
|
||||
self.passwords[netloc] = (username, password)
|
||||
|
||||
assert (
|
||||
# Credentials were found
|
||||
(username is not None and password is not None)
|
||||
# Credentials were not found
|
||||
or (username is None and password is None)
|
||||
), f"Could not load credentials from url: {original_url}"
|
||||
|
||||
return url, username, password
|
||||
|
||||
def __call__(self, req: PreparedRequest) -> PreparedRequest:
|
||||
# Get credentials for this request
|
||||
assert req.url is not None
|
||||
url, username, password = self._get_url_and_credentials(req.url)
|
||||
|
||||
# Set the url of the request to the url without any credentials
|
||||
req.url = url
|
||||
|
||||
if username is not None and password is not None:
|
||||
# Send the basic auth with this request
|
||||
req = HTTPBasicAuth(username, password)(req)
|
||||
|
||||
# Attach a hook to handle 401 responses
|
||||
req.register_hook("response", self.handle_401)
|
||||
|
||||
return req
|
||||
|
||||
# Factored out to allow for easy patching in tests
|
||||
def _prompt_for_password(self, netloc: str) -> tuple[str | None, str | None, bool]:
|
||||
username = ask_input(f"User for {netloc}: ") if self.prompting else None
|
||||
if not username:
|
||||
return None, None, False
|
||||
if self.use_keyring:
|
||||
auth = self._get_keyring_auth(netloc, username)
|
||||
if auth and auth[0] is not None and auth[1] is not None:
|
||||
return auth[0], auth[1], False
|
||||
password = ask_password("Password: ")
|
||||
return username, password, True
|
||||
|
||||
# Factored out to allow for easy patching in tests
|
||||
def _should_save_password_to_keyring(self) -> bool:
|
||||
if (
|
||||
not self.prompting
|
||||
or not self.use_keyring
|
||||
or not self.keyring_provider.has_keyring
|
||||
):
|
||||
return False
|
||||
return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"
|
||||
|
||||
def handle_401(self, resp: Response, **kwargs: Any) -> Response:
|
||||
# We only care about 401 responses, anything else we want to just
|
||||
# pass through the actual response
|
||||
if resp.status_code != 401:
|
||||
return resp
|
||||
|
||||
username, password = None, None
|
||||
|
||||
# Query the keyring for credentials:
|
||||
if self.use_keyring:
|
||||
username, password = self._get_new_credentials(
|
||||
resp.url,
|
||||
allow_netrc=False,
|
||||
allow_keyring=True,
|
||||
)
|
||||
|
||||
# We are not able to prompt the user so simply return the response
|
||||
if not self.prompting and not username and not password:
|
||||
return resp
|
||||
|
||||
parsed = urllib.parse.urlparse(resp.url)
|
||||
|
||||
# Prompt the user for a new username and password
|
||||
save = False
|
||||
if not username and not password:
|
||||
username, password, save = self._prompt_for_password(parsed.netloc)
|
||||
|
||||
# Store the new username and password to use for future requests
|
||||
self._credentials_to_save = None
|
||||
if username is not None and password is not None:
|
||||
self.passwords[parsed.netloc] = (username, password)
|
||||
|
||||
# Prompt to save the password to keyring
|
||||
if save and self._should_save_password_to_keyring():
|
||||
self._credentials_to_save = Credentials(
|
||||
url=parsed.netloc,
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
|
||||
# Consume content and release the original connection to allow our new
|
||||
# request to reuse the same one.
|
||||
# The result of the assignment isn't used, it's just needed to consume
|
||||
# the content.
|
||||
_ = resp.content
|
||||
resp.raw.release_conn()
|
||||
|
||||
# Add our new username and password to the request
|
||||
req = HTTPBasicAuth(username or "", password or "")(resp.request)
|
||||
req.register_hook("response", self.warn_on_401)
|
||||
|
||||
# On successful request, save the credentials that were used to
|
||||
# keyring. (Note that if the user responded "no" above, this member
|
||||
# is not set and nothing will be saved.)
|
||||
if self._credentials_to_save:
|
||||
req.register_hook("response", self.save_credentials)
|
||||
|
||||
# Send our new request
|
||||
new_resp = resp.connection.send(req, **kwargs)
|
||||
new_resp.history.append(resp)
|
||||
|
||||
return new_resp
|
||||
|
||||
def warn_on_401(self, resp: Response, **kwargs: Any) -> None:
|
||||
"""Response callback to warn about incorrect credentials."""
|
||||
if resp.status_code == 401:
|
||||
logger.warning(
|
||||
"401 Error, Credentials not correct for %s",
|
||||
resp.request.url,
|
||||
)
|
||||
|
||||
def save_credentials(self, resp: Response, **kwargs: Any) -> None:
|
||||
"""Response callback to save credentials on success."""
|
||||
assert (
|
||||
self.keyring_provider.has_keyring
|
||||
), "should never reach here without keyring"
|
||||
|
||||
creds = self._credentials_to_save
|
||||
self._credentials_to_save = None
|
||||
if creds and resp.status_code < 400:
|
||||
try:
|
||||
logger.info("Saving credentials to keyring")
|
||||
self.keyring_provider.save_auth_info(
|
||||
creds.url, creds.username, creds.password
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to save credentials")
|
||||
128
venv/Lib/site-packages/pip/_internal/network/cache.py
Normal file
128
venv/Lib/site-packages/pip/_internal/network/cache.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""HTTP cache implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from typing import Any, BinaryIO, Callable
|
||||
|
||||
from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
|
||||
from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
|
||||
from pip._vendor.requests.models import Response
|
||||
|
||||
from pip._internal.utils.filesystem import (
|
||||
adjacent_tmp_file,
|
||||
copy_directory_permissions,
|
||||
replace,
|
||||
)
|
||||
from pip._internal.utils.misc import ensure_dir
|
||||
|
||||
|
||||
def is_from_cache(response: Response) -> bool:
|
||||
return getattr(response, "from_cache", False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def suppressed_cache_errors() -> Generator[None, None, None]:
|
||||
"""If we can't access the cache then we can just skip caching and process
|
||||
requests as if caching wasn't enabled.
|
||||
"""
|
||||
try:
|
||||
yield
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class SafeFileCache(SeparateBodyBaseCache):
|
||||
"""
|
||||
A file based cache which is safe to use even when the target directory may
|
||||
not be accessible or writable.
|
||||
|
||||
There is a race condition when two processes try to write and/or read the
|
||||
same entry at the same time, since each entry consists of two separate
|
||||
files (https://github.com/psf/cachecontrol/issues/324). We therefore have
|
||||
additional logic that makes sure that both files to be present before
|
||||
returning an entry; this fixes the read side of the race condition.
|
||||
|
||||
For the write side, we assume that the server will only ever return the
|
||||
same data for the same URL, which ought to be the case for files pip is
|
||||
downloading. PyPI does not have a mechanism to swap out a wheel for
|
||||
another wheel, for example. If this assumption is not true, the
|
||||
CacheControl issue will need to be fixed.
|
||||
"""
|
||||
|
||||
def __init__(self, directory: str) -> None:
|
||||
assert directory is not None, "Cache directory must not be None."
|
||||
super().__init__()
|
||||
self.directory = directory
|
||||
|
||||
def _get_cache_path(self, name: str) -> str:
|
||||
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
|
||||
# class for backwards-compatibility and to avoid using a non-public
|
||||
# method.
|
||||
hashed = SeparateBodyFileCache.encode(name)
|
||||
parts = list(hashed[:5]) + [hashed]
|
||||
return os.path.join(self.directory, *parts)
|
||||
|
||||
def get(self, key: str) -> bytes | None:
|
||||
# The cache entry is only valid if both metadata and body exist.
|
||||
metadata_path = self._get_cache_path(key)
|
||||
body_path = metadata_path + ".body"
|
||||
if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
|
||||
return None
|
||||
with suppressed_cache_errors():
|
||||
with open(metadata_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
def _write_to_file(self, path: str, writer_func: Callable[[BinaryIO], Any]) -> None:
|
||||
"""Common file writing logic with proper permissions and atomic replacement."""
|
||||
with suppressed_cache_errors():
|
||||
ensure_dir(os.path.dirname(path))
|
||||
|
||||
with adjacent_tmp_file(path) as f:
|
||||
writer_func(f)
|
||||
# Inherit the read/write permissions of the cache directory
|
||||
# to enable multi-user cache use-cases.
|
||||
copy_directory_permissions(self.directory, f)
|
||||
|
||||
replace(f.name, path)
|
||||
|
||||
def _write(self, path: str, data: bytes) -> None:
|
||||
self._write_to_file(path, lambda f: f.write(data))
|
||||
|
||||
def _write_from_io(self, path: str, source_file: BinaryIO) -> None:
|
||||
self._write_to_file(path, lambda f: shutil.copyfileobj(source_file, f))
|
||||
|
||||
def set(
|
||||
self, key: str, value: bytes, expires: int | datetime | None = None
|
||||
) -> None:
|
||||
path = self._get_cache_path(key)
|
||||
self._write(path, value)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
path = self._get_cache_path(key)
|
||||
with suppressed_cache_errors():
|
||||
os.remove(path)
|
||||
with suppressed_cache_errors():
|
||||
os.remove(path + ".body")
|
||||
|
||||
def get_body(self, key: str) -> BinaryIO | None:
|
||||
# The cache entry is only valid if both metadata and body exist.
|
||||
metadata_path = self._get_cache_path(key)
|
||||
body_path = metadata_path + ".body"
|
||||
if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
|
||||
return None
|
||||
with suppressed_cache_errors():
|
||||
return open(body_path, "rb")
|
||||
|
||||
def set_body(self, key: str, body: bytes) -> None:
|
||||
path = self._get_cache_path(key) + ".body"
|
||||
self._write(path, body)
|
||||
|
||||
def set_body_from_io(self, key: str, body_file: BinaryIO) -> None:
|
||||
"""Set the body of the cache entry from a file object."""
|
||||
path = self._get_cache_path(key) + ".body"
|
||||
self._write_from_io(path, body_file)
|
||||
341
venv/Lib/site-packages/pip/_internal/network/download.py
Normal file
341
venv/Lib/site-packages/pip/_internal/network/download.py
Normal file
@@ -0,0 +1,341 @@
|
||||
"""Download files with progress indicators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import email.message
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import BinaryIO
|
||||
|
||||
from pip._vendor.requests import PreparedRequest
|
||||
from pip._vendor.requests.models import Response
|
||||
from pip._vendor.urllib3 import HTTPResponse as URLlib3Response
|
||||
from pip._vendor.urllib3._collections import HTTPHeaderDict
|
||||
from pip._vendor.urllib3.exceptions import ReadTimeoutError
|
||||
|
||||
from pip._internal.cli.progress_bars import BarType, get_download_progress_renderer
|
||||
from pip._internal.exceptions import IncompleteDownloadError, NetworkConnectionError
|
||||
from pip._internal.models.index import PyPI
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.network.cache import SafeFileCache, is_from_cache
|
||||
from pip._internal.network.session import CacheControlAdapter, PipSession
|
||||
from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
|
||||
from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_http_response_size(resp: Response) -> int | None:
|
||||
try:
|
||||
return int(resp.headers["content-length"])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _get_http_response_etag_or_last_modified(resp: Response) -> str | None:
|
||||
"""
|
||||
Return either the ETag or Last-Modified header (or None if neither exists).
|
||||
The return value can be used in an If-Range header.
|
||||
"""
|
||||
return resp.headers.get("etag", resp.headers.get("last-modified"))
|
||||
|
||||
|
||||
def _log_download(
|
||||
resp: Response,
|
||||
link: Link,
|
||||
progress_bar: BarType,
|
||||
total_length: int | None,
|
||||
range_start: int | None = 0,
|
||||
) -> Iterable[bytes]:
|
||||
if link.netloc == PyPI.file_storage_domain:
|
||||
url = link.show_url
|
||||
else:
|
||||
url = link.url_without_fragment
|
||||
|
||||
logged_url = redact_auth_from_url(url)
|
||||
|
||||
if total_length:
|
||||
if range_start:
|
||||
logged_url = (
|
||||
f"{logged_url} ({format_size(range_start)}/{format_size(total_length)})"
|
||||
)
|
||||
else:
|
||||
logged_url = f"{logged_url} ({format_size(total_length)})"
|
||||
|
||||
if is_from_cache(resp):
|
||||
logger.info("Using cached %s", logged_url)
|
||||
elif range_start:
|
||||
logger.info("Resuming download %s", logged_url)
|
||||
else:
|
||||
logger.info("Downloading %s", logged_url)
|
||||
|
||||
if logger.getEffectiveLevel() > logging.INFO:
|
||||
show_progress = False
|
||||
elif is_from_cache(resp):
|
||||
show_progress = False
|
||||
elif not total_length:
|
||||
show_progress = True
|
||||
elif total_length > (512 * 1024):
|
||||
show_progress = True
|
||||
else:
|
||||
show_progress = False
|
||||
|
||||
chunks = response_chunks(resp)
|
||||
|
||||
if not show_progress:
|
||||
return chunks
|
||||
|
||||
renderer = get_download_progress_renderer(
|
||||
bar_type=progress_bar, size=total_length, initial_progress=range_start
|
||||
)
|
||||
return renderer(chunks)
|
||||
|
||||
|
||||
def sanitize_content_filename(filename: str) -> str:
|
||||
"""
|
||||
Sanitize the "filename" value from a Content-Disposition header.
|
||||
"""
|
||||
return os.path.basename(filename)
|
||||
|
||||
|
||||
def parse_content_disposition(content_disposition: str, default_filename: str) -> str:
|
||||
"""
|
||||
Parse the "filename" value from a Content-Disposition header, and
|
||||
return the default filename if the result is empty.
|
||||
"""
|
||||
m = email.message.Message()
|
||||
m["content-type"] = content_disposition
|
||||
filename = m.get_param("filename")
|
||||
if filename:
|
||||
# We need to sanitize the filename to prevent directory traversal
|
||||
# in case the filename contains ".." path parts.
|
||||
filename = sanitize_content_filename(str(filename))
|
||||
return filename or default_filename
|
||||
|
||||
|
||||
def _get_http_response_filename(resp: Response, link: Link) -> str:
|
||||
"""Get an ideal filename from the given HTTP response, falling back to
|
||||
the link filename if not provided.
|
||||
"""
|
||||
filename = link.filename # fallback
|
||||
# Have a look at the Content-Disposition header for a better guess
|
||||
content_disposition = resp.headers.get("content-disposition")
|
||||
if content_disposition:
|
||||
filename = parse_content_disposition(content_disposition, filename)
|
||||
ext: str | None = splitext(filename)[1]
|
||||
if not ext:
|
||||
ext = mimetypes.guess_extension(resp.headers.get("content-type", ""))
|
||||
if ext:
|
||||
filename += ext
|
||||
if not ext and link.url != resp.url:
|
||||
ext = os.path.splitext(resp.url)[1]
|
||||
if ext:
|
||||
filename += ext
|
||||
return filename
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FileDownload:
|
||||
"""Stores the state of a single link download."""
|
||||
|
||||
link: Link
|
||||
output_file: BinaryIO
|
||||
size: int | None
|
||||
bytes_received: int = 0
|
||||
reattempts: int = 0
|
||||
|
||||
def is_incomplete(self) -> bool:
|
||||
return bool(self.size is not None and self.bytes_received < self.size)
|
||||
|
||||
def write_chunk(self, data: bytes) -> None:
|
||||
self.bytes_received += len(data)
|
||||
self.output_file.write(data)
|
||||
|
||||
def reset_file(self) -> None:
|
||||
"""Delete any saved data and reset progress to zero."""
|
||||
self.output_file.seek(0)
|
||||
self.output_file.truncate()
|
||||
self.bytes_received = 0
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(
|
||||
self,
|
||||
session: PipSession,
|
||||
progress_bar: BarType,
|
||||
) -> None:
|
||||
self._session = session
|
||||
self._progress_bar = progress_bar
|
||||
self._resume_retries = session.resume_retries
|
||||
assert (
|
||||
self._resume_retries >= 0
|
||||
), "Number of max resume retries must be bigger or equal to zero"
|
||||
|
||||
def batch(
|
||||
self, links: Iterable[Link], location: str
|
||||
) -> Iterable[tuple[Link, tuple[str, str]]]:
|
||||
"""Convenience method to download multiple links."""
|
||||
for link in links:
|
||||
filepath, content_type = self(link, location)
|
||||
yield link, (filepath, content_type)
|
||||
|
||||
def __call__(self, link: Link, location: str) -> tuple[str, str]:
|
||||
"""Download a link and save it under location."""
|
||||
resp = self._http_get(link)
|
||||
download_size = _get_http_response_size(resp)
|
||||
|
||||
filepath = os.path.join(location, _get_http_response_filename(resp, link))
|
||||
with open(filepath, "wb") as content_file:
|
||||
download = _FileDownload(link, content_file, download_size)
|
||||
self._process_response(download, resp)
|
||||
if download.is_incomplete():
|
||||
self._attempt_resumes_or_redownloads(download, resp)
|
||||
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
return filepath, content_type
|
||||
|
||||
def _process_response(self, download: _FileDownload, resp: Response) -> None:
|
||||
"""Download and save chunks from a response."""
|
||||
chunks = _log_download(
|
||||
resp,
|
||||
download.link,
|
||||
self._progress_bar,
|
||||
download.size,
|
||||
range_start=download.bytes_received,
|
||||
)
|
||||
try:
|
||||
for chunk in chunks:
|
||||
download.write_chunk(chunk)
|
||||
except ReadTimeoutError as e:
|
||||
# If the download size is not known, then give up downloading the file.
|
||||
if download.size is None:
|
||||
raise e
|
||||
|
||||
logger.warning("Connection timed out while downloading.")
|
||||
|
||||
def _attempt_resumes_or_redownloads(
|
||||
self, download: _FileDownload, first_resp: Response
|
||||
) -> None:
|
||||
"""Attempt to resume/restart the download if connection was dropped."""
|
||||
|
||||
while download.reattempts < self._resume_retries and download.is_incomplete():
|
||||
assert download.size is not None
|
||||
download.reattempts += 1
|
||||
logger.warning(
|
||||
"Attempting to resume incomplete download (%s/%s, attempt %d)",
|
||||
format_size(download.bytes_received),
|
||||
format_size(download.size),
|
||||
download.reattempts,
|
||||
)
|
||||
|
||||
try:
|
||||
resume_resp = self._http_get_resume(download, should_match=first_resp)
|
||||
# Fallback: if the server responded with 200 (i.e., the file has
|
||||
# since been modified or range requests are unsupported) or any
|
||||
# other unexpected status, restart the download from the beginning.
|
||||
must_restart = resume_resp.status_code != HTTPStatus.PARTIAL_CONTENT
|
||||
if must_restart:
|
||||
download.reset_file()
|
||||
download.size = _get_http_response_size(resume_resp)
|
||||
first_resp = resume_resp
|
||||
|
||||
self._process_response(download, resume_resp)
|
||||
except (ConnectionError, ReadTimeoutError, OSError):
|
||||
continue
|
||||
|
||||
# No more resume attempts. Raise an error if the download is still incomplete.
|
||||
if download.is_incomplete():
|
||||
os.remove(download.output_file.name)
|
||||
raise IncompleteDownloadError(download)
|
||||
|
||||
# If we successfully completed the download via resume, manually cache it
|
||||
# as a complete response to enable future caching
|
||||
if download.reattempts > 0:
|
||||
self._cache_resumed_download(download, first_resp)
|
||||
|
||||
def _cache_resumed_download(
|
||||
self, download: _FileDownload, original_response: Response
|
||||
) -> None:
|
||||
"""
|
||||
Manually cache a file that was successfully downloaded via resume retries.
|
||||
|
||||
cachecontrol doesn't cache 206 (Partial Content) responses, since they
|
||||
are not complete files. This method manually adds the final file to the
|
||||
cache as though it was downloaded in a single request, so that future
|
||||
requests can use the cache.
|
||||
"""
|
||||
url = download.link.url_without_fragment
|
||||
adapter = self._session.get_adapter(url)
|
||||
|
||||
# Check if the adapter is the CacheControlAdapter (i.e. caching is enabled)
|
||||
if not isinstance(adapter, CacheControlAdapter):
|
||||
logger.debug(
|
||||
"Skipping resume download caching: no cache controller for %s", url
|
||||
)
|
||||
return
|
||||
|
||||
# Check SafeFileCache is being used
|
||||
assert isinstance(
|
||||
adapter.cache, SafeFileCache
|
||||
), "separate body cache not in use!"
|
||||
|
||||
synthetic_request = PreparedRequest()
|
||||
synthetic_request.prepare(method="GET", url=url, headers={})
|
||||
|
||||
synthetic_response_headers = HTTPHeaderDict()
|
||||
for key, value in original_response.headers.items():
|
||||
if key.lower() not in ["content-range", "content-length"]:
|
||||
synthetic_response_headers[key] = value
|
||||
synthetic_response_headers["content-length"] = str(download.size)
|
||||
|
||||
synthetic_response = URLlib3Response(
|
||||
body="",
|
||||
headers=synthetic_response_headers,
|
||||
status=200,
|
||||
preload_content=False,
|
||||
)
|
||||
|
||||
# Save metadata and then stream the file contents to cache.
|
||||
cache_url = adapter.controller.cache_url(url)
|
||||
metadata_blob = adapter.controller.serializer.dumps(
|
||||
synthetic_request, synthetic_response, b""
|
||||
)
|
||||
adapter.cache.set(cache_url, metadata_blob)
|
||||
download.output_file.flush()
|
||||
with open(download.output_file.name, "rb") as f:
|
||||
adapter.cache.set_body_from_io(cache_url, f)
|
||||
|
||||
logger.debug(
|
||||
"Cached resumed download as complete response for future use: %s", url
|
||||
)
|
||||
|
||||
def _http_get_resume(
|
||||
self, download: _FileDownload, should_match: Response
|
||||
) -> Response:
|
||||
"""Issue a HTTP range request to resume the download."""
|
||||
# To better understand the download resumption logic, see the mdn web docs:
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Range_requests
|
||||
headers = HEADERS.copy()
|
||||
headers["Range"] = f"bytes={download.bytes_received}-"
|
||||
# If possible, use a conditional range request to avoid corrupted
|
||||
# downloads caused by the remote file changing in-between.
|
||||
if identifier := _get_http_response_etag_or_last_modified(should_match):
|
||||
headers["If-Range"] = identifier
|
||||
return self._http_get(download.link, headers)
|
||||
|
||||
def _http_get(self, link: Link, headers: Mapping[str, str] = HEADERS) -> Response:
|
||||
target_url = link.url_without_fragment
|
||||
try:
|
||||
resp = self._session.get(target_url, headers=headers, stream=True)
|
||||
raise_for_status(resp)
|
||||
except NetworkConnectionError as e:
|
||||
assert e.response is not None
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s", e.response.status_code, link
|
||||
)
|
||||
raise
|
||||
return resp
|
||||
215
venv/Lib/site-packages/pip/_internal/network/lazy_wheel.py
Normal file
215
venv/Lib/site-packages/pip/_internal/network/lazy_wheel.py
Normal file
@@ -0,0 +1,215 @@
|
||||
"""Lazy ZIP over HTTP"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"]
|
||||
|
||||
from bisect import bisect_left, bisect_right
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any
|
||||
from zipfile import BadZipFile, ZipFile
|
||||
|
||||
from pip._vendor.packaging.utils import NormalizedName
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
|
||||
from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution
|
||||
from pip._internal.network.session import PipSession
|
||||
from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
|
||||
|
||||
|
||||
class HTTPRangeRequestUnsupported(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def dist_from_wheel_url(
|
||||
name: NormalizedName, url: str, session: PipSession
|
||||
) -> BaseDistribution:
|
||||
"""Return a distribution object from the given wheel URL.
|
||||
|
||||
This uses HTTP range requests to only fetch the portion of the wheel
|
||||
containing metadata, just enough for the object to be constructed.
|
||||
If such requests are not supported, HTTPRangeRequestUnsupported
|
||||
is raised.
|
||||
"""
|
||||
with LazyZipOverHTTP(url, session) as zf:
|
||||
# For read-only ZIP files, ZipFile only needs methods read,
|
||||
# seek, seekable and tell, not the whole IO protocol.
|
||||
wheel = MemoryWheel(zf.name, zf) # type: ignore
|
||||
# After context manager exit, wheel.name
|
||||
# is an invalid file by intention.
|
||||
return get_wheel_distribution(wheel, name)
|
||||
|
||||
|
||||
class LazyZipOverHTTP:
|
||||
"""File-like object mapped to a ZIP file over HTTP.
|
||||
|
||||
This uses HTTP range requests to lazily fetch the file's content,
|
||||
which is supposed to be fed to ZipFile. If such requests are not
|
||||
supported by the server, raise HTTPRangeRequestUnsupported
|
||||
during initialization.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, url: str, session: PipSession, chunk_size: int = CONTENT_CHUNK_SIZE
|
||||
) -> None:
|
||||
head = session.head(url, headers=HEADERS)
|
||||
raise_for_status(head)
|
||||
assert head.status_code == 200
|
||||
self._session, self._url, self._chunk_size = session, url, chunk_size
|
||||
self._length = int(head.headers["Content-Length"])
|
||||
self._file = NamedTemporaryFile()
|
||||
self.truncate(self._length)
|
||||
self._left: list[int] = []
|
||||
self._right: list[int] = []
|
||||
if "bytes" not in head.headers.get("Accept-Ranges", "none"):
|
||||
raise HTTPRangeRequestUnsupported("range request is not supported")
|
||||
self._check_zip()
|
||||
|
||||
@property
|
||||
def mode(self) -> str:
|
||||
"""Opening mode, which is always rb."""
|
||||
return "rb"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Path to the underlying file."""
|
||||
return self._file.name
|
||||
|
||||
def seekable(self) -> bool:
|
||||
"""Return whether random access is supported, which is True."""
|
||||
return True
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the file."""
|
||||
self._file.close()
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
"""Whether the file is closed."""
|
||||
return self._file.closed
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
"""Read up to size bytes from the object and return them.
|
||||
|
||||
As a convenience, if size is unspecified or -1,
|
||||
all bytes until EOF are returned. Fewer than
|
||||
size bytes may be returned if EOF is reached.
|
||||
"""
|
||||
download_size = max(size, self._chunk_size)
|
||||
start, length = self.tell(), self._length
|
||||
stop = length if size < 0 else min(start + download_size, length)
|
||||
start = max(0, stop - download_size)
|
||||
self._download(start, stop - 1)
|
||||
return self._file.read(size)
|
||||
|
||||
def readable(self) -> bool:
|
||||
"""Return whether the file is readable, which is True."""
|
||||
return True
|
||||
|
||||
def seek(self, offset: int, whence: int = 0) -> int:
|
||||
"""Change stream position and return the new absolute position.
|
||||
|
||||
Seek to offset relative position indicated by whence:
|
||||
* 0: Start of stream (the default). pos should be >= 0;
|
||||
* 1: Current position - pos may be negative;
|
||||
* 2: End of stream - pos usually negative.
|
||||
"""
|
||||
return self._file.seek(offset, whence)
|
||||
|
||||
def tell(self) -> int:
|
||||
"""Return the current position."""
|
||||
return self._file.tell()
|
||||
|
||||
def truncate(self, size: int | None = None) -> int:
|
||||
"""Resize the stream to the given size in bytes.
|
||||
|
||||
If size is unspecified resize to the current position.
|
||||
The current stream position isn't changed.
|
||||
|
||||
Return the new file size.
|
||||
"""
|
||||
return self._file.truncate(size)
|
||||
|
||||
def writable(self) -> bool:
|
||||
"""Return False."""
|
||||
return False
|
||||
|
||||
def __enter__(self) -> LazyZipOverHTTP:
|
||||
self._file.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc: Any) -> None:
|
||||
self._file.__exit__(*exc)
|
||||
|
||||
@contextmanager
|
||||
def _stay(self) -> Generator[None, None, None]:
|
||||
"""Return a context manager keeping the position.
|
||||
|
||||
At the end of the block, seek back to original position.
|
||||
"""
|
||||
pos = self.tell()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.seek(pos)
|
||||
|
||||
def _check_zip(self) -> None:
|
||||
"""Check and download until the file is a valid ZIP."""
|
||||
end = self._length - 1
|
||||
for start in reversed(range(0, end, self._chunk_size)):
|
||||
self._download(start, end)
|
||||
with self._stay():
|
||||
try:
|
||||
# For read-only ZIP files, ZipFile only needs
|
||||
# methods read, seek, seekable and tell.
|
||||
ZipFile(self)
|
||||
except BadZipFile:
|
||||
pass
|
||||
else:
|
||||
break
|
||||
|
||||
def _stream_response(
|
||||
self, start: int, end: int, base_headers: dict[str, str] = HEADERS
|
||||
) -> Response:
|
||||
"""Return HTTP response to a range request from start to end."""
|
||||
headers = base_headers.copy()
|
||||
headers["Range"] = f"bytes={start}-{end}"
|
||||
# TODO: Get range requests to be correctly cached
|
||||
headers["Cache-Control"] = "no-cache"
|
||||
return self._session.get(self._url, headers=headers, stream=True)
|
||||
|
||||
def _merge(
|
||||
self, start: int, end: int, left: int, right: int
|
||||
) -> Generator[tuple[int, int], None, None]:
|
||||
"""Return a generator of intervals to be fetched.
|
||||
|
||||
Args:
|
||||
start (int): Start of needed interval
|
||||
end (int): End of needed interval
|
||||
left (int): Index of first overlapping downloaded data
|
||||
right (int): Index after last overlapping downloaded data
|
||||
"""
|
||||
lslice, rslice = self._left[left:right], self._right[left:right]
|
||||
i = start = min([start] + lslice[:1])
|
||||
end = max([end] + rslice[-1:])
|
||||
for j, k in zip(lslice, rslice):
|
||||
if j > i:
|
||||
yield i, j - 1
|
||||
i = k + 1
|
||||
if i <= end:
|
||||
yield i, end
|
||||
self._left[left:right], self._right[left:right] = [start], [end]
|
||||
|
||||
def _download(self, start: int, end: int) -> None:
|
||||
"""Download bytes from start to end inclusively."""
|
||||
with self._stay():
|
||||
left = bisect_left(self._right, start)
|
||||
right = bisect_right(self._left, end)
|
||||
for start, end in self._merge(start, end, left, right):
|
||||
response = self._stream_response(start, end)
|
||||
response.raise_for_status()
|
||||
self.seek(start)
|
||||
for chunk in response_chunks(response, self._chunk_size):
|
||||
self._file.write(chunk)
|
||||
Reference in New Issue
Block a user