Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ END_UNRELEASED_TEMPLATE
* Python toolchain from [20260414] release.
* (pypi) `package_metadata` support, fixes
[#2054](https://github.com/bazel-contrib/rules_python/issues/2054).
* (pypi) Added {attr}`pip.parse.restrict_visibility_to` to expose only
packages listed in requirement files while keeping lockfile transitive
dependencies available internally. Fixes
[#3413](https://github.com/bazel-contrib/rules_python/issues/3413).

[20260325]: https://github.com/astral-sh/python-build-standalone/releases/tag/20260325
[20260414]: https://github.com/astral-sh/python-build-standalone/releases/tag/20260414
Expand Down
26 changes: 24 additions & 2 deletions docs/pypi/download.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,30 @@ pip.parse(
use_repo(pip, "my_deps")
```

For more documentation, see the Bzlmod examples under the {gh-path}`examples` folder or the documentation
for the {obj}`@rules_python//python/extensions:pip.bzl` extension.
For more documentation, see the Bzlmod examples under the
{gh-path}`examples` folder or the documentation for the
{obj}`@rules_python//python/extensions:pip.bzl` extension.

## Restricting exposed hub packages

By default, every package in {attr}`pip.parse.requirements_lock` gets a public
hub alias, such as `@my_deps//foo`. If you want only direct dependencies to be
available to user code, set {attr}`pip.parse.restrict_visibility_to` to one or
more requirement files that list those direct packages:

```starlark
pip.parse(
hub_name = "my_deps",
python_version = "3.13",
requirements_lock = "//:requirements_lock.txt",
restrict_visibility_to = ["//:requirements.in"],
)
```

Packages in the lock file that are not listed in the restricted requirement
files still get generated wheel repositories, so direct dependencies can use
their transitive dependencies. Their hub aliases are visible only to the
generated wheel repositories and are not public targets for user code.

:::note}
We are using a host-platform compatible toolchain by default to setup pip dependencies.
Expand Down
15 changes: 15 additions & 0 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,21 @@ The Python version the dependencies are targetting, in Major.Minor format
If an interpreter isn't explicitly provided (using `python_interpreter` or
`python_interpreter_target`), then the version specified here must have
a corresponding `python.toolchain()` configured.
""",
),
"restrict_visibility_to": attr.label_list(
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like adding this, but I imaging that we could repurpose this set of files for more labels.

  1. Restrict visibility.
  2. Create a @pypi//:lock.update target automatically that includes the srcs passed here as srcs passed to the lock file. This plumbing would be similar to what rules_jvm have with their repin.

We should also assume that we will need to support pyproject.toml files/file via the same interface in the future.

allow_files = True,
doc = """
A list of requirement files whose package names are exposed as public hub
targets. Packages in the lock files that are not listed here still get wheel
repositories so they can be used as transitive dependencies, but their hub
aliases are only visible to repositories generated by this `pip.parse` hub.

This is useful when your lock file contains transitive dependencies that should
remain implementation details of your direct dependencies.

:::{versionadded} VERSION_NEXT_FEATURE
:::
""",
),
"simpleapi_skip": attr.string_list(
Expand Down
1 change: 1 addition & 0 deletions python/private/pypi/hub_builder.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,7 @@ def _create_whl_repos(
extra_pip_args = pip_attr.extra_pip_args,
get_index_urls = self._get_index_urls.get(pip_attr.python_version),
evaluate_markers = _evaluate_markers(self, pip_attr),
exposed_requirements = pip_attr.restrict_visibility_to,
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we create a separate function for this?

    exposed_requirements = parse_dep_srcs(module_ctx, pip_attr.restrict_visibility_to)

That way we make the code more maintainable and easier to test.

logger = logger,
)

Expand Down
6 changes: 4 additions & 2 deletions python/private/pypi/hub_repository.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ exports_files(["requirements.bzl"])
"""

def _impl(rctx):
bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys()
bzl_packages = rctx.attr.packages
aliases = render_multiplatform_pkg_aliases(
aliases = {
key: _whl_config_settings_from_json(values)
for key, values in rctx.attr.whl_map.items()
},
exposed_packages = bzl_packages,
extra_hub_aliases = rctx.attr.extra_hub_aliases,
requirement_cycles = rctx.attr.groups,
platform_config_settings = rctx.attr.platform_config_settings,
Expand Down Expand Up @@ -81,7 +82,8 @@ hub_repository = repository_rule(
"packages": attr.string_list(
mandatory = False,
doc = """\
The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys.
The list of packages that will be exposed via public hub aliases and
all_*requirements macros.
""",
),
"platform_config_settings": attr.string_list_dict(
Expand Down
43 changes: 39 additions & 4 deletions python/private/pypi/parse_requirements.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def parse_requirements(
platforms = {},
get_index_urls = None,
evaluate_markers = None,
exposed_requirements = [],
extract_url_srcs = True,
logger):
"""Get the requirements with platforms that the requirements apply to.
Expand All @@ -62,6 +63,9 @@ def parse_requirements(
the platforms stored as values in the input dict. Returns the same
dict, but with values being platforms that are compatible with the
requirements line.
exposed_requirements: List of requirements files. When present, only
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we key this by_platform as well?

packages listed in these files should be exposed via the hub
repository.
extract_url_srcs: A boolean to enable extracting URLs from requirement
lines to enable using bazel downloader.
logger: repo_utils.logger, a simple struct to log diagnostic messages.
Expand Down Expand Up @@ -92,6 +96,7 @@ def parse_requirements(
reqs_with_env_markers = {}
index_url = None
extra_index_urls = []
exposed_package_names = _exposed_package_names(ctx, exposed_requirements)
for file, plats in requirements_by_platform.items():
logger.trace(lambda: "Using {} for {}".format(file, plats))
contents = ctx.read(file)
Expand Down Expand Up @@ -231,17 +236,34 @@ def parse_requirements(
# for p in dist.target_platforms
# ]

normalized_name = normalize_name(name)
is_exposed = len(requirement_target_platforms) == len(requirements)
if exposed_package_names != None and normalized_name not in exposed_package_names:
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't like the double negatives, maybe the following is easier to read?

Suggested change
if exposed_package_names != None and normalized_name not in exposed_package_names:
if not (exposed_package_names and normalized_name in exposed_package_names):

is_exposed = False
Comment on lines +241 to +242
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like this logic to be somewhere outside the parse_requirements.bzl file. Would it be possible to move it to hub_builder.bzl? The output of parse_requirements have this is_exposed flag (that is an internal implementation detail) and if we make the lock sources mandatory, then we would be able to drop that implementation detail, which actually makes the code a little bit more complex than it needs to be.


item = struct(
# Return normalized names
name = normalize_name(name),
is_exposed = len(requirement_target_platforms) == len(requirements),
name = normalized_name,
is_exposed = is_exposed,
is_multiple_versions = len(reqs.values()) > 1,
index_url = pkg_sources.index_url if pkg_sources else "",
srcs = package_srcs,
)
ret.append(item)
if not item.is_exposed and logger:
logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format(
if (
exposed_package_names != None and
normalized_name not in exposed_package_names and
logger
):
logger.trace(lambda: (
"Package '{}' will not be exposed because it is not present " +
"in restrict_visibility_to"
).format(name))
if len(requirement_target_platforms) != len(requirements) and logger:
logger.trace(lambda: (
"Package '{}' will not be exposed because it is only present " +
"on a subset of platforms: {} out of {}"
).format(
name,
sorted(requirement_target_platforms),
sorted(requirements),
Expand All @@ -251,6 +273,19 @@ def parse_requirements(

return ret

def _exposed_package_names(ctx, exposed_requirements):
"""Parse the requirement files that define hub-exposed package names."""
if not exposed_requirements:
return None

exposed = {}
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This acts like a union. Is this the right thing to do here?

From the modeling perspective, the requirements may be different based on the target platform. That said, I think this approach is OK in the parse_requirements internals. Since we are asking the user to provide the source lock files.

for file in exposed_requirements:
parse_result = parse_requirements_txt(ctx.read(file))
for distribution, _ in parse_result.requirements:
exposed[normalize_name(distribution)] = None

return exposed

def _package_srcs(
*,
name,
Expand Down
53 changes: 50 additions & 3 deletions python/private/pypi/render_pkg_aliases.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def _repr_actual(aliases):
else:
return render.dict(aliases, key_repr = _repr_config_setting)

def _render_common_aliases(*, name, aliases, **kwargs):
def _render_common_aliases(*, name, aliases, visibility, **kwargs):
pkg_aliases = render.call(
"pkg_aliases",
name = repr(name),
Expand All @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs):
return """\
load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases")
{extra_loads}
package(default_visibility = ["//visibility:public"])
package(default_visibility = {visibility})

{aliases}""".format(
aliases = pkg_aliases,
extra_loads = extra_loads,
visibility = render.list(visibility),
)

def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs):
def render_pkg_aliases(
*,
aliases,
requirement_cycles = None,
extra_hub_aliases = {},
exposed_packages = None,
**kwargs):
"""Create alias declarations for each PyPI package.

The aliases should be appended to the pip_repository BUILD.bazel file. These aliases
Expand All @@ -100,6 +107,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
requirement_cycles: any package groups to also add.
extra_hub_aliases: The list of extra aliases for each whl to be added
in addition to the default ones.
exposed_packages: The public hub packages. When present, other packages
are only visible to generated wheel repositories.
**kwargs: Extra kwargs to pass to the rules.

Returns:
Expand All @@ -124,12 +133,20 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
for whl_name in group_whls
}

exposed_packages = _normalize_package_names(exposed_packages)
internal_visibility = _internal_visibility(aliases)

files = {
"{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases(
name = normalize_name(name),
aliases = pkg_aliases,
extra_aliases = extra_hub_aliases.get(normalize_name(name), []),
group_name = whl_group_mapping.get(normalize_name(name)),
visibility = _package_visibility(
name = normalize_name(name),
exposed_packages = exposed_packages,
internal_visibility = internal_visibility,
),
**kwargs
).strip()
for name, pkg_aliases in aliases.items()
Expand All @@ -139,6 +156,36 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles)
return files

def _normalize_package_names(packages):
if packages == None:
return None

return {
normalize_name(package): None
for package in packages
}

def _internal_visibility(aliases):
repo_names = {}
for pkg_aliases in aliases.values():
if type(pkg_aliases) == type(""):
repo_names[pkg_aliases] = None
continue

for repo_name in pkg_aliases.values():
repo_names[repo_name] = None

return [
"@{}//:__pkg__".format(repo_name)
for repo_name in sorted(repo_names)
]

def _package_visibility(*, name, exposed_packages, internal_visibility):
if exposed_packages == None or name in exposed_packages:
return ["//visibility:public"]

return internal_visibility

def _major_minor(python_version):
major, _, tail = python_version.partition(".")
minor, _, _ = tail.partition(".")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ def _test_exec_matches_target_python_version(name):
)

# This is never matched. It's just here so that toolchains from the
# environment don't match.
# environment don't match. Keep the target settings mismatched so this
# unconstrained toolchain cannot satisfy host execution platforms.
native.toolchain(
name = "99_target_default",
toolchain_type = TARGET_TOOLCHAIN_TYPE,
Expand Down Expand Up @@ -119,6 +120,7 @@ def _test_exec_matches_target_python_version(name):
name = "99_exec_default",
toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE,
toolchain = ":exec_default",
target_settings = ["//python/config_settings:is_python_3.11"],
)

analysis_test(
Expand Down
2 changes: 2 additions & 0 deletions tests/pypi/extension/pip_parse.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def pip_parse(
requirements_linux = None,
requirements_lock = None,
requirements_windows = None,
restrict_visibility_to = [],
target_platforms = [],
simpleapi_skip = [],
timeout = 600,
Expand Down Expand Up @@ -60,6 +61,7 @@ def pip_parse(
requirements_linux = requirements_linux,
requirements_lock = requirements_lock,
requirements_windows = requirements_windows,
restrict_visibility_to = restrict_visibility_to,
timeout = timeout,
whl_modifications = whl_modifications,
parallel_download = False,
Expand Down
59 changes: 59 additions & 0 deletions tests/pypi/hub_builder/hub_builder_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,65 @@ def _test_simple(env):

_tests.append(_test_simple)

def _test_restrict_visibility_to(env):
builder = hub_builder(env)
builder.pip_parse(
_mock_mctx(
os_name = "osx",
arch_name = "aarch64",
mock_files = {
"requirements.in": "foo>=0.0.1\n",
"requirements.txt": """\
foo==0.0.1 --hash=sha256:deadbeef
dep-of-foo==0.0.1 --hash=sha256:deadb00f
""",
},
),
_parse(
hub_name = "pypi",
python_version = "3.15",
requirements_lock = "requirements.txt",
restrict_visibility_to = ["requirements.in"],
),
)
pypi = builder.build()

pypi.exposed_packages().contains_exactly(["foo"])
pypi.group_map().contains_exactly({})
pypi.whl_map().contains_exactly({
"dep_of_foo": {
"pypi_315_dep_of_foo": [
whl_config_setting(
version = "3.15",
),
],
},
"foo": {
"pypi_315_foo": [
whl_config_setting(
version = "3.15",
),
],
},
})
pypi.whl_libraries().contains_exactly({
"pypi_315_dep_of_foo": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "dep-of-foo==0.0.1 --hash=sha256:deadb00f",
},
"pypi_315_foo": {
"config_load": "@pypi//:config.bzl",
"dep_template": "@pypi//{name}:{target}",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "foo==0.0.1 --hash=sha256:deadbeef",
},
})
pypi.extra_aliases().contains_exactly({})

_tests.append(_test_restrict_visibility_to)

def _test_simple_multiple_requirements(env):
sub_tests = {
("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f",
Expand Down
Loading