From 7b5df74bb15753d88d952a02ebb6c8b714a6b0cd Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 16:45:56 -0400 Subject: [PATCH 01/47] chore(java-pubsub): improve monorepo migration tooling Updates migration automation scripts as part of the effort to migrate java-pubsub into the monorepo. Refines generation_config.yaml updating logic to prevent data loss, patches root POM sorting to preserve original formatting, and adapts CI transformers to maintain standard Clirr execution steps. --- monorepo-migration/migrate-pubsub.sh | 45 +++++++++ monorepo-migration/transform_workflow.py | 30 +++++- .../update_generation_config.py | 99 +++++-------------- monorepo-migration/update_root_pom.py | 3 +- 4 files changed, 101 insertions(+), 76 deletions(-) create mode 100755 monorepo-migration/migrate-pubsub.sh diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh new file mode 100755 index 000000000000..71c88351ff68 --- /dev/null +++ b/monorepo-migration/migrate-pubsub.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error +set -e + +# Get absolute paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "========================================================" +echo " Staging java-pubsub migration" +echo "========================================================" + +# 1. Configure environment for the base migrate.sh script +export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" +export MIGRATION_HEAD_BRANCH="main" +export SQUASH_COMMITS="false" +export CODEOWNER="@googleapis/pubsub-team" + +# 2. Execute the central migration script +# This performs git read-tree, POM modernization, workflow transformation, and generation config updates. +# Note: migrate.sh works in an isolated sibling clone to avoid polluting the active workspace. +"${SCRIPT_DIR}/migrate.sh" + +echo "" +echo "========================================================" +echo "Migration staged successfully!" +echo "Results are available in the isolated clone:" +echo " ../../migration-work/google-cloud-java-target" +echo "Current Branch: migrate-java-pubsub" +echo "Next Steps: cd ../../migration-work/google-cloud-java-target && mvn clean install -DskipTests" +echo "========================================================" diff --git a/monorepo-migration/transform_workflow.py b/monorepo-migration/transform_workflow.py index fba791634ebe..e1b5398aa813 100644 --- a/monorepo-migration/transform_workflow.py +++ b/monorepo-migration/transform_workflow.py @@ -33,11 +33,33 @@ def transform(content, lib_name): library: - '{lib_name}/**'""" + clirr_template = f""" clirr: + needs: filter + if: ${{{{ needs.filter.outputs.library == 'true' }}}} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr + BUILD_SUBDIR: {lib_name}""" + in_jobs = False skip_current_job = False current_job_is_windows = False + skip_lines_count = 0 + for line in lines: + if skip_lines_count > 0: + skip_lines_count -= 1 + continue + if line.startswith('name:') and not in_jobs: name_match = re.match(r'^name:\s*(.*)', line) if name_match: @@ -62,11 +84,11 @@ def transform(content, lib_name): if job_match: job_name = job_match.group(1) current_job_is_windows = False # Reset for new job + skip_current_job = False if job_name == 'clirr': skip_current_job = True + new_lines.extend(clirr_template.splitlines()) continue - else: - skip_current_job = False if job_name != 'filter': new_lines.append(line) @@ -84,6 +106,10 @@ def transform(content, lib_name): new_lines.append(" run: git config --system core.longpaths true") continue + if 'name: Support longpaths' in line and current_job_is_windows: + skip_lines_count = 1 + continue + if 'run: echo "SUREFIRE_JVM_OPT=' in line and '!java17' not in line: line = line.replace('" >> $GITHUB_ENV', ' -P !java17" >> $GITHUB_ENV') if 'build.bat' in line: diff --git a/monorepo-migration/update_generation_config.py b/monorepo-migration/update_generation_config.py index 1ba7e6fc8c3e..2ab0c49f4b57 100644 --- a/monorepo-migration/update_generation_config.py +++ b/monorepo-migration/update_generation_config.py @@ -15,87 +15,40 @@ import sys import yaml -import re -def get_library_id(lib): - """ - Returns a unique identifier for a library. - Prefer 'library_name', then 'api_shortname'. - """ - if 'library_name' in lib: - return f"java-{lib['library_name']}" - if 'api_shortname' in lib: - return f"java-{lib['api_shortname']}" - return "unknown" - -def merge_libraries(target_libs, source_libs): +def update_config(target_path, source_path): """ - Merges source_libs into target_libs. - Libraries are matched by get_library_id. - GAPICs are merged and deduplicated by proto_path. - The final list is sorted by library_id. + Appends the library configuration from the source_path to the target_path. + This avoids rewriting the entire target YAML, preserving all comments and ordering. """ - # Map from library_id to library dict - target_map = {get_library_id(lib): lib for lib in target_libs} - - for s_lib in source_libs: - lib_id = get_library_id(s_lib) - - # Clean up source library (remove repo fields) - s_lib_cleaned = {k: v for k, v in s_lib.items() if k not in ('repo', 'repo_short')} - - if lib_id in target_map: - t_lib = target_map[lib_id] - # Merge GAPICs - t_gapics_list = t_lib.get('GAPICs', []) - s_gapics_list = s_lib_cleaned.get('GAPICs', []) - - # Map by proto_path for deduplication - proto_map = {g['proto_path']: g for g in t_gapics_list} - for g in s_gapics_list: - proto_map[g['proto_path']] = g - - # Sort GAPICs by proto_path - sorted_protos = sorted(proto_map.keys()) - t_lib['GAPICs'] = [proto_map[p] for p in sorted_protos] - - # Update other fields from source - for k, v in s_lib_cleaned.items(): - if k != 'GAPICs': - t_lib[k] = v - else: - target_map[lib_id] = s_lib_cleaned - - # Return sorted list of libraries - sorted_ids = sorted(target_map.keys()) - return [target_map[lib_id] for lib_id in sorted_ids] - -def update_config(target_path, source_path): - with open(target_path, 'r') as f: - target_content = f.read() - with open(source_path, 'r') as f: source_data = yaml.safe_load(f) or {} - # Load target data - target_data = yaml.safe_load(target_content) or {} - - target_libs = target_data.get('libraries', []) source_libs = source_data.get('libraries', []) - - merged_libs = merge_libraries(target_libs, source_libs) - target_data['libraries'] = merged_libs + if not source_libs: + print("No libraries found in source config.") + return - # Write back - with open(target_path, 'w') as f: - # Check if there was a license header in the original file - header_match = re.search(r'^(#.*?\n\n)', target_content, re.DOTALL) - if header_match: - f.write(header_match.group(1)) - - # Use yaml.dump to write the data. - # sort_keys=False to preserve order of fields within libraries if possible (YAML 1.2+ usually does, but pyyaml depends) - yaml.dump(target_data, f, sort_keys=False, default_flow_style=False, indent=2) + # In standalone repos, there is usually only one library definition. + new_libs = [] + for s_lib in source_libs: + # Clean up source library (remove repo fields as they are now internal to monorepo) + if 'repo' in s_lib: + del s_lib['repo'] + if 'repo_short' in s_lib: + del s_lib['repo_short'] + new_libs.append(s_lib) + + # Dump the new library entries as a YAML string + # sort_keys=False preserves the field ordering (e.g., api_shortname first) + new_yaml_str = yaml.dump(new_libs, sort_keys=False, default_flow_style=False, indent=2) + + # Append to the existing monorepo generation_config.yaml + # This ensures we don't rewrite the file and lose comments/ordering. + with open(target_path, 'a') as f: + f.write(new_yaml_str.rstrip() + "\n") + + print(f"Appended {len(new_libs)} libraries to {target_path}") if __name__ == "__main__": if len(sys.argv) != 3: diff --git a/monorepo-migration/update_root_pom.py b/monorepo-migration/update_root_pom.py index fec12930dee3..dd1203ae28a0 100644 --- a/monorepo-migration/update_root_pom.py +++ b/monorepo-migration/update_root_pom.py @@ -14,6 +14,7 @@ # limitations under the License. import sys +import re def update_root_pom(pom_path, module_name): new_module = f' {module_name}\n' @@ -39,7 +40,7 @@ def update_root_pom(pom_path, module_name): java_lines = lines[start_java:end_java] if not any(f'{module_name}' in l for l in java_lines): java_lines.append(new_module) - java_lines.sort() + java_lines.sort(key=lambda line: m.group(1) if '' in line and (m := re.search(r'(.*?)', line)) else line) lines = lines[:start_java] + java_lines + lines[end_java:] else: if not any(f'{module_name}' in l for l in lines): From 110e1dbb23f60c2044cf064e0a6bd07cab746729 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 17:54:00 -0400 Subject: [PATCH 02/47] chore(java-pubsub): inject SHAs and fetch-depth for lint jobs --- monorepo-migration/transform_workflow.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/monorepo-migration/transform_workflow.py b/monorepo-migration/transform_workflow.py index e1b5398aa813..d83548d913c7 100644 --- a/monorepo-migration/transform_workflow.py +++ b/monorepo-migration/transform_workflow.py @@ -52,6 +52,7 @@ def transform(content, lib_name): in_jobs = False skip_current_job = False current_job_is_windows = False + current_job_is_lint = False skip_lines_count = 0 @@ -84,6 +85,7 @@ def transform(content, lib_name): if job_match: job_name = job_match.group(1) current_job_is_windows = False # Reset for new job + current_job_is_lint = (job_name == 'lint') skip_current_job = False if job_name == 'clirr': skip_current_job = True @@ -110,6 +112,18 @@ def transform(content, lib_name): skip_lines_count = 1 continue + if '- uses: actions/checkout' in line and current_job_is_lint: + new_lines.append(" - uses: actions/checkout@v4") + new_lines.append(" with:") + new_lines.append(" fetch-depth: 0") + continue + + if 'JOB_TYPE: lint' in line and current_job_is_lint: + new_lines.append(line) + new_lines.append(" HEAD_SHA: ${{ github.event.pull_request.head.sha }}") + new_lines.append(" BASE_SHA: ${{ github.event.pull_request.base.sha }}") + continue + if 'run: echo "SUREFIRE_JVM_OPT=' in line and '!java17' not in line: line = line.replace('" >> $GITHUB_ENV', ' -P !java17" >> $GITHUB_ENV') if 'build.bat' in line: From b7038a38b89bfb02b0f906da149201295a03bb1e Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 18:16:01 -0400 Subject: [PATCH 03/47] chore(java-pubsub): add conditional version check to core migration script --- monorepo-migration/migrate.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 93b4b2c788c4..143d2bee2219 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,6 +348,20 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.9b Conditionally skip version check if unmanaged dependencies exist +echo "Verifying non-release-please version compliance..." +if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then + echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." + sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || \\\\\n [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + + echo "Committing linter adjustment..." + git add generation/check_non_release_please_versions.sh + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): skip version check for $SOURCE_REPO_NAME" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +else + echo "All dependency versions fully managed. No linter adjustments required." +fi + # 7.11 Verify compilation echo "Verifying compilation..." BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh From ed02927aeeefca86648c684b4bb7a0df1956a4a6 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 23:33:19 -0400 Subject: [PATCH 04/47] impl(java-pubsub): dynamically exempt unmanaged modules from global matrix --- monorepo-migration/migrate.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 143d2bee2219..76172c170a81 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,11 +348,20 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.8c Exempt module from global integration testing matrix +echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." +sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" + +echo "Committing common.sh update..." +git add .kokoro/common.sh +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): exempt from global integration testing matrix" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + # 7.9b Conditionally skip version check if unmanaged dependencies exist echo "Verifying non-release-please version compliance..." if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." - sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || \\\\\n [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" echo "Committing linter adjustment..." git add generation/check_non_release_please_versions.sh From 0d91612fe7f9ee40a4bf6301b441a117c80d3ed2 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:44:12 -0400 Subject: [PATCH 05/47] chore: integrate version alignment and owlbot formatting into migrate script --- monorepo-migration/migrate.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 76172c170a81..34adcafce989 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,6 +348,18 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.12b Align all version markers across the monorepo +echo "Aligning all version markers using apply_versions.sh..." +bash generation/apply_versions.sh versions.txt current + +# 7.12c Sync all owlbot.py formatting +echo "Syncing all owlbot.py formatting..." +bash generation/update_owlbot_postprocessor_config.sh || true + +git add -u +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): align versions and format owlbot configurations" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + # 7.8c Exempt module from global integration testing matrix echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" From 19d4690c9c0370e541e12f8406c3acfe87f72c16 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:45:20 -0400 Subject: [PATCH 06/47] chore: allow MIGRATION_HEAD_BRANCH override in migrate-pubsub.sh --- monorepo-migration/migrate-pubsub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh index 71c88351ff68..e5062bbfbd23 100755 --- a/monorepo-migration/migrate-pubsub.sh +++ b/monorepo-migration/migrate-pubsub.sh @@ -26,7 +26,7 @@ echo "========================================================" # 1. Configure environment for the base migrate.sh script export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" -export MIGRATION_HEAD_BRANCH="main" +export MIGRATION_HEAD_BRANCH="${MIGRATION_HEAD_BRANCH:-main}" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/pubsub-team" From c33bb07117232a170c0102b88dbac16f50e35b8e Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:47:01 -0400 Subject: [PATCH 07/47] chore: revert MIGRATION_HEAD_BRANCH to main --- monorepo-migration/migrate-pubsub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh index e5062bbfbd23..71c88351ff68 100755 --- a/monorepo-migration/migrate-pubsub.sh +++ b/monorepo-migration/migrate-pubsub.sh @@ -26,7 +26,7 @@ echo "========================================================" # 1. Configure environment for the base migrate.sh script export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" -export MIGRATION_HEAD_BRANCH="${MIGRATION_HEAD_BRANCH:-main}" +export MIGRATION_HEAD_BRANCH="main" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/pubsub-team" From 052436929d455feab7bb07e283be6701ab47fecd Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 12:13:48 -0400 Subject: [PATCH 08/47] chore: migrate GraalVM Native presubmit config in migration script --- monorepo-migration/migrate.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 34adcafce989..de23f1e49332 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -173,6 +173,32 @@ git merge --allow-unrelated-histories --no-ff "$SOURCE_REPO_NAME/main" -s ours - echo "Reading tree into prefix $SOURCE_REPO_NAME/..." git read-tree --prefix="$SOURCE_REPO_NAME/" -u "$SOURCE_REPO_NAME/main" +# 6.3 Commit the initial import +echo "Committing initial import of $SOURCE_REPO_NAME..." +git add "$SOURCE_REPO_NAME" +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + +# 6.4b Migrate GraalVM Native presubmit config if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" ]; then + echo "Migrating graalvm-native-a.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg..." + mkdir -p .kokoro/presubmit + sed -e 's/value: "graalvm"/value: "graalvm-single"/' \ + "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + + # Append BUILD_SUBDIR + cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate GraalVM Native presubmit config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.5 Remove common files from the root of the migrated library echo "Removing common files from the root of $SOURCE_REPO_NAME/..." rm -f "$SOURCE_REPO_NAME/.gitignore" @@ -180,16 +206,14 @@ rm -f "$SOURCE_REPO_NAME/renovate.json" rm -f "$SOURCE_REPO_NAME/LICENSE" rm -f "$SOURCE_REPO_NAME/java.header" rm -rf "$SOURCE_REPO_NAME/.kokoro" -# rm -rf "$SOURCE_REPO_NAME/.kokoro/continuous" "$SOURCE_REPO_NAME/.kokoro/nightly" "$SOURCE_REPO_NAME/.kokoro/presubmit" rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" -delete -# 7. Commit the migration -echo "Committing migration..." +echo "Committing removal of common files..." git add "$SOURCE_REPO_NAME" -git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): remove common files from module root" COMMIT_COUNT=$((COMMIT_COUNT + 1)) # 7.1 Update CODEOWNERS From 319b79dc61e725d30d70679bad86b544c996f033 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 14:23:40 -0400 Subject: [PATCH 09/47] chore: migrate GraalVM and Integration presubmit configs in migration script --- monorepo-migration/migrate.sh | 23 +++++++++++- .../update_linter_exclusions.py | 37 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 monorepo-migration/update_linter_exclusions.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index de23f1e49332..b63b013042c7 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -56,6 +56,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -199,6 +200,26 @@ EOF COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 6.4c Migrate Integration presubmit config if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" ]; then + echo "Migrating integration.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg..." + mkdir -p .kokoro/presubmit + sed -e 's/value: "integration"/value: "integration-single"/' \ + "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + + # Append BUILD_SUBDIR + cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate Integration presubmit config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.5 Remove common files from the root of the migrated library echo "Removing common files from the root of $SOURCE_REPO_NAME/..." rm -f "$SOURCE_REPO_NAME/.gitignore" @@ -397,7 +418,7 @@ COMMIT_COUNT=$((COMMIT_COUNT + 1)) echo "Verifying non-release-please version compliance..." if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." - sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + python3 "$UPDATE_LINTER_EXCLUSIONS_SCRIPT" "generation/check_non_release_please_versions.sh" "$SOURCE_REPO_NAME" echo "Committing linter adjustment..." git add generation/check_non_release_please_versions.sh diff --git a/monorepo-migration/update_linter_exclusions.py b/monorepo-migration/update_linter_exclusions.py new file mode 100644 index 000000000000..e0cad10b1c39 --- /dev/null +++ b/monorepo-migration/update_linter_exclusions.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +def update_linter(file_path, lib_name): + with open(file_path, 'r') as f: + content = f.read() + + target = ' [[ "${pomFile}" =~ .*java-vertexai.* ]] || \\' + replacement = f' [[ "${{pomFile}}" =~ .*{lib_name}.* ]] || \\\n{target}' + + if target in content: + content = content.replace(target, replacement) + with open(file_path, 'w') as f: + f.write(content) + print(f"Successfully added {lib_name} to linter exclusions.") + else: + print(f"Target not found in {file_path}") + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} file_path lib_name") + sys.exit(1) + update_linter(sys.argv[1], sys.argv[2]) From e9afc54b6f6df2aa2a68ac551cca604fe4d9a7e9 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 20:45:51 -0400 Subject: [PATCH 10/47] chore: preserve GEMINI.md and DEVELOPMENT.md during migration --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index b63b013042c7..6c2fdc703631 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -230,7 +230,7 @@ rm -rf "$SOURCE_REPO_NAME/.kokoro" rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" -find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" -delete +find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" ! -name "GEMINI.md" ! -name "DEVELOPMENT.md" -delete echo "Committing removal of common files..." git add "$SOURCE_REPO_NAME" From ca74740bfa1cbe204af385996c4b263b18bb49e5 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 21:03:28 -0400 Subject: [PATCH 11/47] chore: add update_ci_filters.py and integrate with migrate script --- monorepo-migration/migrate.sh | 6 ++-- monorepo-migration/update_ci_filters.py | 42 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 monorepo-migration/update_ci_filters.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 6c2fdc703631..4de82e80e62f 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -57,6 +57,7 @@ UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config. UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" +UPDATE_CI_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_ci_filters.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -408,9 +409,10 @@ COMMIT_COUNT=$((COMMIT_COUNT + 1)) # 7.8c Exempt module from global integration testing matrix echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" +python3 "$UPDATE_CI_FILTERS_SCRIPT" ".github/workflows/ci.yaml" "$SOURCE_REPO_NAME" -echo "Committing common.sh update..." -git add .kokoro/common.sh +echo "Committing common.sh and ci.yaml updates..." +git add .kokoro/common.sh .github/workflows/ci.yaml git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): exempt from global integration testing matrix" COMMIT_COUNT=$((COMMIT_COUNT + 1)) diff --git a/monorepo-migration/update_ci_filters.py b/monorepo-migration/update_ci_filters.py new file mode 100644 index 000000000000..2a9704cdf838 --- /dev/null +++ b/monorepo-migration/update_ci_filters.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re + +def update_ci_filters(ci_yaml, lib_name): + with open(ci_yaml, 'r') as f: + content = f.read() + + # match `!(...)` pattern in ci.yaml + pattern = r'!\(([^)]+)\)' + + def repl(match): + modules = match.group(1).split('|') + if lib_name not in modules: + modules.append(lib_name) + modules.sort() + return '!(' + '|'.join(modules) + ')' + + new_content = re.sub(pattern, repl, content) + with open(ci_yaml, 'w') as f: + f.write(new_content) + print(f"Successfully added {lib_name} to CI exclusions in {ci_yaml}.") + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + update_ci_filters(sys.argv[1], sys.argv[2]) From 211ca9b588ed5dc2f7e88066cd860c00889f10a4 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 26 Mar 2026 18:13:28 +0000 Subject: [PATCH 12/47] feat: add script to transfer issues --- monorepo-migration/migrate_issues.sh | 129 +++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 monorepo-migration/migrate_issues.sh diff --git a/monorepo-migration/migrate_issues.sh b/monorepo-migration/migrate_issues.sh new file mode 100755 index 000000000000..79604bc46a3a --- /dev/null +++ b/monorepo-migration/migrate_issues.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# migrate_issues.sh - Transfer open GitHub issues from one repo to another using gh CLI. + +set -e + +usage() { + echo "Usage: $0 [--label