diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d04eb4bc4e0..9af1a5882ce2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,6 +112,7 @@ jobs: outputs: packages: ${{ steps.filter.outputs.changes }} steps: + - uses: actions/checkout@v4 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: @@ -338,8 +339,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} enforcer: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/java-bigtable-ci.yaml b/.github/workflows/java-bigtable-ci.yaml index ef6db6e93a0d..c0328b1435ce 100644 --- a/.github/workflows/java-bigtable-ci.yaml +++ b/.github/workflows/java-bigtable-ci.yaml @@ -137,8 +137,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} clirr: needs: filter if: ${{ needs.filter.outputs.library == 'true' }} diff --git a/.github/workflows/java-pubsub-ci.yaml b/.github/workflows/java-pubsub-ci.yaml index 5c87fa56c2f0..c4723c36b6f0 100644 --- a/.github/workflows/java-pubsub-ci.yaml +++ b/.github/workflows/java-pubsub-ci.yaml @@ -137,8 +137,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} clirr: needs: filter if: ${{ needs.filter.outputs.library == 'true' }} diff --git a/.github/workflows/java-spanner-jdbc-ci.yaml b/.github/workflows/java-spanner-jdbc-ci.yaml index 5d114841a7d2..4cbb5184193a 100644 --- a/.github/workflows/java-spanner-jdbc-ci.yaml +++ b/.github/workflows/java-spanner-jdbc-ci.yaml @@ -141,8 +141,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} required: needs: [ units, units-java8, windows, dependencies, javadoc, lint ] name: conditional-required-check diff --git a/.github/workflows/java-storage-nio-ci.yaml b/.github/workflows/java-storage-nio-ci.yaml index e53a05764dcb..efddd8ac27fd 100644 --- a/.github/workflows/java-storage-nio-ci.yaml +++ b/.github/workflows/java-storage-nio-ci.yaml @@ -141,8 +141,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} required: needs: [ units, units-java8, windows, dependencies, javadoc, lint ] name: conditional-required-check diff --git a/generation/apply_versions.sh b/generation/apply_versions.sh index 99d1ab3a3109..826d6df2213b 100755 --- a/generation/apply_versions.sh +++ b/generation/apply_versions.sh @@ -25,7 +25,7 @@ if [[ "$column_name" == "released" ]]; then column_index=2 elif [[ "$column_name" == "current" ]]; then column_index=3 -elif "$column_name" != "current" ]]; then +elif [[ "$column_name" != "current" ]]; then echo "Error: column_name must be either 'released' or 'current'" exit 1 fi diff --git a/generation/update_owlbot_postprocessor_config.sh b/generation/update_owlbot_postprocessor_config.sh index a59d865bc2d9..275041f01305 100755 --- a/generation/update_owlbot_postprocessor_config.sh +++ b/generation/update_owlbot_postprocessor_config.sh @@ -6,7 +6,10 @@ set -e -for dir in $(find . -mindepth 2 -maxdepth 2 -name owlbot.py | grep -v 'java-common-protos/' | grep -v 'java-iam/' | grep -v 'java-showcase/' | sort | xargs dirname ); do +TARGET_MODULE="${1:-.}" + +for owlbot_script in $(find "$TARGET_MODULE" -name owlbot.py | grep -v 'java-common-protos/' | grep -v 'java-iam/' | grep -v 'java-showcase/' | sort); do + dir=$(dirname "$owlbot_script") pushd "$dir" # form a perl command to replace java.common_templates() invocation diff --git a/monorepo-migration/.gitignore b/monorepo-migration/.gitignore new file mode 100644 index 000000000000..ae0b0d95a4b2 --- /dev/null +++ b/monorepo-migration/.gitignore @@ -0,0 +1,3 @@ +.git-filter-repo/ +__pycache__/ +*.pyc diff --git a/monorepo-migration/migrate-bigtable.sh b/monorepo-migration/migrate-bigtable.sh new file mode 100755 index 000000000000..076d93ec1b06 --- /dev/null +++ b/monorepo-migration/migrate-bigtable.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error +set -e + +# Get absolute paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "========================================================" +echo " Staging java-bigtable migration" +echo "========================================================" + +# 1. Configure environment for the base migrate.sh script +export SOURCE_REPO_URL="https://github.com/googleapis/java-bigtable" +export MIGRATION_HEAD_BRANCH="main" +export SQUASH_COMMITS="false" +export CODEOWNER="@googleapis/bigtable-team" +export BOM_SUBSTITUTIONS="gapic-libraries-bom:google-cloud-monitoring-bom" +export PRE_INSTALL_DEPS="java-monitoring/google-cloud-monitoring-bom,java-monitoring/google-cloud-monitoring" + +# 2. Execute the central migration script +"${SCRIPT_DIR}/migrate.sh" + +echo "" +echo "========================================================" +echo "Migration staged successfully!" +echo "Results are available in the isolated clone:" +echo " migration-work/google-cloud-java-target" +echo "Current Branch: migrate-java-bigtable" +echo "Next Steps: cd migration-work/google-cloud-java-target && mvn clean install -DskipTests" +echo "========================================================" diff --git a/monorepo-migration/migrate-firestore.sh b/monorepo-migration/migrate-firestore.sh new file mode 100755 index 000000000000..babdce93fd58 --- /dev/null +++ b/monorepo-migration/migrate-firestore.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error +set -e + +# Get absolute paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "========================================================" +echo " Staging java-firestore migration" +echo "========================================================" + +# 1. Configure environment for the base migrate.sh script +export SOURCE_REPO_URL="https://github.com/googleapis/java-firestore" +export MIGRATION_HEAD_BRANCH="main" +export SQUASH_COMMITS="false" +export CODEOWNER="@googleapis/firestore-team" + +# 2. Execute the central migration script +"${SCRIPT_DIR}/migrate.sh" + +echo "" +echo "========================================================" +echo "Migration staged successfully!" +echo "Results are available in the isolated clone:" +echo " migration-work/google-cloud-java-target" +echo "Current Branch: migrate-java-firestore" +echo "Next Steps: cd migration-work/google-cloud-java-target && mvn clean install -DskipTests" +echo "========================================================" diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh new file mode 100755 index 000000000000..3d8a3d1cda54 --- /dev/null +++ b/monorepo-migration/migrate-pubsub.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error +set -e + +# Get absolute paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "========================================================" +echo " Staging java-pubsub migration" +echo "========================================================" + +# 1. Configure environment for the base migrate.sh script +export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" +export MIGRATION_HEAD_BRANCH="main" +export SQUASH_COMMITS="false" +export CODEOWNER="@googleapis/pubsub-team" +export SUREFIRE_JVM_OPT="-DskipTests" + +# 2. Execute the central migration script +# This performs git read-tree, POM modernization, workflow transformation, and generation config updates. +# Note: migrate.sh works in an isolated sibling clone to avoid polluting the active workspace. +"${SCRIPT_DIR}/migrate.sh" + +echo "" +echo "========================================================" +echo "Migration staged successfully!" +echo "Results are available in the isolated clone:" +echo " ../../migration-work/google-cloud-java-target" +echo "Current Branch: migrate-java-pubsub" +echo "Next Steps: cd ../../migration-work/google-cloud-java-target && mvn clean install -DskipTests" +echo "========================================================" diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 93b4b2c788c4..b4c795e0805d 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -17,6 +17,26 @@ # Exit on error set -e +# Get absolute path to the script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +# Ensure git-filter-repo is available in PATH +if ! command -v git-filter-repo >/dev/null 2>&1; then + LOCAL_FILTER_REPO_DIR="${SCRIPT_DIR}/.git-filter-repo" + LOCAL_FILTER_REPO="${LOCAL_FILTER_REPO_DIR}/git-filter-repo" + + if [ ! -f "$LOCAL_FILTER_REPO" ]; then + echo "git-filter-repo not found locally or in PATH. Downloading version v2.45.0..." + mkdir -p "$LOCAL_FILTER_REPO_DIR" + curl -sSL -o "$LOCAL_FILTER_REPO" "https://raw.githubusercontent.com/newren/git-filter-repo/v2.45.0/git-filter-repo" + chmod +x "$LOCAL_FILTER_REPO" + fi + + echo "Injecting local .git-filter-repo into PATH..." + export PATH="${LOCAL_FILTER_REPO_DIR}:$PATH" +fi + # Function to check if a command exists check_command() { if ! command -v "$1" >/dev/null 2>&1; then @@ -29,6 +49,7 @@ check_command() { check_command git check_command python3 check_command mvn +check_command git-filter-repo # Configuration MONOREPO_URL="https://github.com/googleapis/google-cloud-java" @@ -42,13 +63,12 @@ CODEOWNER="${CODEOWNER:-}" SOURCE_REPO_NAME="${SOURCE_REPO_URL##*/}" MONOREPO_NAME="${MONOREPO_URL##*/}" -# Use a temporary working directory sibling to the current monorepo -WORKING_DIR="../../migration-work" +# Use a temporary working directory sibling to the current monorepo, anchored to script location +WORKING_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)/migration-work" SOURCE_DIR="$WORKING_DIR/$SOURCE_REPO_NAME-source" TARGET_DIR="$WORKING_DIR/$MONOREPO_NAME-target" -# Get absolute path to the transformation script before any cd -TRANSFORM_SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TRANSFORM_SCRIPT_DIR="${SCRIPT_DIR}" TRANSFORM_SCRIPT="$TRANSFORM_SCRIPT_DIR/transform_workflow.py" MODERNIZE_POM_SCRIPT="$TRANSFORM_SCRIPT_DIR/modernize_pom.py" UPDATE_ROOT_POM_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_root_pom.py" @@ -56,6 +76,10 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +UPDATE_SAMPLES_BUILD_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_samples_build.py" +UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" +UPDATE_CI_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_ci_filters.py" +UPDATE_CHANGES_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_changes_filters.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -65,23 +89,28 @@ echo "Starting migration using git read-tree with isolated clones..." # 0. Create working directory mkdir -p "$WORKING_DIR" -MIGRATION_HEAD_BRANCH="${MIGRATION_HEAD_BRANCH:-main}" +MIGRATION_HEAD_BRANCH="main" echo "Basing migration branch on: ${MIGRATION_HEAD_BRANCH}" # 1. Clone the source repository -if [ ! -d "$SOURCE_DIR" ]; then - echo "Cloning source repo: $SOURCE_REPO_URL into $SOURCE_DIR" - git clone "$SOURCE_REPO_URL" "$SOURCE_DIR" +if [ "${SKIP_SOURCE_UPDATE:-false}" = "true" ] && [ -d "$SOURCE_DIR" ]; then + echo "Skipping source repository update and reusing existing directory..." else - echo "Source directory $SOURCE_DIR already exists. Ensuring it is clean and up-to-date..." + echo "Ensuring clean slate for history filters by removing existing source directory..." + rm -rf "$SOURCE_DIR" + echo "Cloning source repo: $SOURCE_REPO_URL into $SOURCE_DIR" + git clone --branch main --single-branch "$SOURCE_REPO_URL" "$SOURCE_DIR" +fi + +if [ "${SKIP_SOURCE_UPDATE:-false}" != "true" ]; then + # 1.2 Rewrite history of the split repo to move files to the target subdirectory + echo "Moving files to destination path: ${SOURCE_REPO_NAME} in history..." cd "$SOURCE_DIR" - git fetch origin - git checkout -f "main" - git reset --hard origin/main - git clean -fd + git filter-repo --to-subdirectory-filter "${SOURCE_REPO_NAME}" --force cd - > /dev/null fi + # 1.5 Extract CODEOWNERS from source repository as default if [ -z "$CODEOWNER" ]; then echo "Attempting to find default CODEOWNER from source repository..." @@ -114,28 +143,30 @@ fi # 2. Clone the target monorepo (the "isolated clone") if [ ! -d "$TARGET_DIR" ]; then echo "Cloning target monorepo: $MONOREPO_URL into $TARGET_DIR" - git clone "$MONOREPO_URL" "$TARGET_DIR" - git checkout -f "${MIGRATION_HEAD_BRANCH}" - git reset --hard origin/${MIGRATION_HEAD_BRANCH} + git clone --branch main --single-branch --depth 1 "$MONOREPO_URL" "$TARGET_DIR" else echo "Target directory $TARGET_DIR already exists. Ensuring it is clean and up-to-date..." cd "$TARGET_DIR" - git fetch origin - git checkout -f "${MIGRATION_HEAD_BRANCH}" - git reset --hard origin/${MIGRATION_HEAD_BRANCH} + git fetch --depth 1 origin main + git checkout -f "main" + git reset --hard origin/main git clean -fd cd - > /dev/null fi cd "$TARGET_DIR" +if [ "$(pwd)" = "$MONOREPO_ROOT" ]; then + echo "CRITICAL ERROR: Script failed to change directory or attempted to run destructive Git operations inside the active workspace!" >&2 + exit 1 +fi # Ensure we are on a clean main branch in the target clone echo "Ensuring clean state in target monorepo..." -git fetch origin +git fetch --depth 1 origin main git reset --hard HEAD git clean -fd -git checkout -f "${MIGRATION_HEAD_BRANCH}" -git reset --hard origin/${MIGRATION_HEAD_BRANCH} +git checkout -f "main" +git reset --hard origin/main git clean -fdx # Check if the repository is already migrated @@ -165,13 +196,119 @@ git remote add "$SOURCE_REPO_NAME" "../$SOURCE_REPO_NAME-source" echo "Fetching $SOURCE_REPO_NAME..." git fetch "$SOURCE_REPO_NAME" -# 5. Merge the histories using 'ours' strategy to keep monorepo content -echo "Merging histories (strategy: ours)..." -git merge --allow-unrelated-histories --no-ff "$SOURCE_REPO_NAME/main" -s ours --no-commit -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +# 5. Merge the histories to pull all rewritten files into their subdirectory directly +echo "Merging histories..." +git merge --allow-unrelated-histories --no-edit --no-gpg-sign "$SOURCE_REPO_NAME/main" -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + +# 6.4b Migrate GraalVM Native presubmit config if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" ]; then + echo "Migrating graalvm-native-a.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME#java-}-graalvm-native-presubmit.cfg..." + mkdir -p .kokoro/presubmit + sed -e 's/value: "graalvm"/value: "graalvm-single"/' \ + "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-graalvm-native-presubmit.cfg" + + # Append BUILD_SUBDIR + cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-graalvm-native-presubmit.cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-graalvm-native-presubmit.cfg" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate GraalVM Native presubmit config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + +# 6.4c Migrate Integration presubmit configurations if present +if ls "$SOURCE_REPO_NAME/.kokoro/presubmit/integration"*.cfg >/dev/null 2>&1; then + mkdir -p .kokoro/presubmit + for cfg_file in "$SOURCE_REPO_NAME/.kokoro/presubmit/integration"*.cfg; do + if [ -f "$cfg_file" ]; then + filename=$(basename "$cfg_file") + new_filename="${filename/integration/${SOURCE_REPO_NAME#java-}-integration}" + target_cfg=".kokoro/presubmit/${new_filename}" + + echo "Migrating and adapting $filename to $target_cfg..." + sed -e 's/value: "integration"/value: "integration-single"/' \ + -e 's/java8/java11/' \ + "$cfg_file" > "$target_cfg" + + # Append BUILD_SUBDIR + cat <> "$target_cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add "$target_cfg" + fi + done + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate Integration presubmit configurations" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + +# 6.4c(2) Migrate custom conformance execution script if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then + echo "Migrating conformance.sh to monorepo root .kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh..." + mkdir -p .kokoro + cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + echo "Adapting conformance script paths and build scopes for monorepo root..." + + # 1. Append popd to the end of the original install block (-T 1C) + sed -i.bak 's|-T 1C|-T 1C\n popd|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + # 2. Construct the unified pre-installation block and pushd subdirectory scoping + if [ -n "${PRE_INSTALL_DEPS}" ]; then + PRE_INSTALL_BLOCK="echo \"Installing toolchain and external prerequisites recursively using monorepo install_modules...\"\ninstall_modules \"${PRE_INSTALL_DEPS}\"\n" + else + PRE_INSTALL_BLOCK="echo \"Installing platform toolchain recursively using monorepo install_modules...\"\ninstall_modules\n" + fi + PRE_INSTALL_BLOCK="${PRE_INSTALL_BLOCK}\npushd ${SOURCE_REPO_NAME}\n# attempt to install 3 times" + + # 3. Inject the pre-installation and pushd block + sed -i.bak "s|# attempt to install 3 times|${PRE_INSTALL_BLOCK}|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + # 4. Adapt other paths to monorepo root + sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|../../test-proxy/known_failures.txt|../../${SOURCE_REPO_NAME}/test-proxy/known_failures.txt|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak 's|mvn install -B -V|mvn install -U -B -V|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak 's|mvn clean install -DskipTests|mvn clean install -U -DskipTests|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + rm -f ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh.bak" + + if [ -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ]; then + echo "Fixing protoc-gen-grpc-java version in test-proxy/pom.xml for Apple Silicon (osx-aarch_64) support..." + sed -i.bak "s|1.24.0:exe:\${os.detected.classifier}|1.62.2:exe:\${os.detected.classifier}|" "${SOURCE_REPO_NAME}/test-proxy/pom.xml" + rm -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml.bak" + + echo "Patching monorepo core .kokoro/build.sh inside target clone to exclude test-proxy from changed-modules linting..." + sed -i.bak 's|unmanaged-dependency-check" \]\] \&\& \\|unmanaged-dependency-check" \]\] \&\& \\\n \[\[ "$(basename "${dir}")" != \*"test-proxy"\* \]\] \&\& \\|' ".kokoro/build.sh" + rm -f ".kokoro/build.sh.bak" + + git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ".kokoro/build.sh" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix test-proxy compilation and exclude from linter changes matrix" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) + fi + + git add ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate and adapt conformance execution script" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi -# 6. Read the tree from the source repo into the desired subdirectory -echo "Reading tree into prefix $SOURCE_REPO_NAME/..." -git read-tree --prefix="$SOURCE_REPO_NAME/" -u "$SOURCE_REPO_NAME/main" +# 6.4d Update repo and repo_short in .repo-metadata.json +REPO_METADATA="$SOURCE_REPO_NAME/.repo-metadata.json" +if [ -f "$REPO_METADATA" ]; then + echo "Updating repo and repo_short in $REPO_METADATA..." + python3 -c "import json; f = '$REPO_METADATA'; d = json.load(open(f)); d['repo'] = 'googleapis/google-cloud-java'; json.dump(d, open(f, 'w'), indent=2); open(f, 'a').write('\n')" + + git add "$REPO_METADATA" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): update .repo-metadata.json" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi # 6.5 Remove common files from the root of the migrated library echo "Removing common files from the root of $SOURCE_REPO_NAME/..." @@ -180,16 +317,14 @@ rm -f "$SOURCE_REPO_NAME/renovate.json" rm -f "$SOURCE_REPO_NAME/LICENSE" rm -f "$SOURCE_REPO_NAME/java.header" rm -rf "$SOURCE_REPO_NAME/.kokoro" -# rm -rf "$SOURCE_REPO_NAME/.kokoro/continuous" "$SOURCE_REPO_NAME/.kokoro/nightly" "$SOURCE_REPO_NAME/.kokoro/presubmit" rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" -find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" -delete +find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" ! -name "GEMINI.md" ! -name "DEVELOPMENT.md" -delete -# 7. Commit the migration -echo "Committing migration..." +echo "Committing removal of common files..." git add "$SOURCE_REPO_NAME" -git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): remove common files from module root" COMMIT_COUNT=$((COMMIT_COUNT + 1)) # 7.1 Update CODEOWNERS @@ -227,7 +362,7 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then case "$filename" in "hermetic_library_generation.yaml" | "update_generation_config.yaml" | \ "approve-readme.yaml" | "auto-release.yaml" | "renovate_config_check.yaml" | \ - "samples.yaml" | "unmanaged_dependency_check.yaml") + "samples.yaml" | "unmanaged_dependency_check.yaml" | "unmanaged-dependency-check.yaml") echo "Skipping redundant workflow: $filename" continue ;; @@ -238,6 +373,10 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then echo "Migrating and adapting $filename to $target_path" python3 "$TRANSFORM_SCRIPT" "$SOURCE_REPO_NAME" < "$workflow" > "$target_path" + sed -i.bak "s|java-version: 8|java-version: 11|" "$target_path" + sed -i.bak "s|github.event.pull_request.head.sha|github.event.pull_request.head.sha \|\| github.sha|g" "$target_path" + sed -i.bak "s|github.event.pull_request.base.sha|github.event.pull_request.base.sha \|\| github.event.before|g" "$target_path" + rm -f "${target_path}.bak" fi done @@ -251,6 +390,18 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 7.5b Adapt samples_build.yaml if present +SAMPLES_BUILD_YAML="$SOURCE_REPO_NAME/.cloudbuild/samples_build.yaml" +if [ -f "$SAMPLES_BUILD_YAML" ]; then + echo "Adapting samples_build.yaml..." + python3 "$UPDATE_SAMPLES_BUILD_SCRIPT" "$SAMPLES_BUILD_YAML" "$SOURCE_REPO_NAME" + + echo "Committing samples_build.yaml update..." + git add "$SAMPLES_BUILD_YAML" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): adapt samples_build.yaml for monorepo" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 7.6 Update generation_config.yaml echo "Updating generation_config.yaml..." SOURCE_CONFIG="$SOURCE_REPO_NAME/generation_config.yaml" @@ -284,8 +435,8 @@ fi # 7.8 Migrate .OwlBot-hermetic.yaml echo "Migrating .OwlBot-hermetic.yaml..." -if [ -f "$SOURCE_DIR/.github/.OwlBot-hermetic.yaml" ]; then - SOURCE_OWLBOT="$SOURCE_DIR/.github/.OwlBot-hermetic.yaml" +if [ -f "$SOURCE_DIR/$SOURCE_REPO_NAME/.github/.OwlBot-hermetic.yaml" ]; then + SOURCE_OWLBOT="$SOURCE_DIR/$SOURCE_REPO_NAME/.github/.OwlBot-hermetic.yaml" else SOURCE_OWLBOT="" fi @@ -303,10 +454,10 @@ fi # 7.8b Migrate owlbot.py echo "Migrating owlbot.py..." -if [ -f "$SOURCE_DIR/owlbot.py" ]; then +if [ -f "$SOURCE_DIR/$SOURCE_REPO_NAME/owlbot.py" ]; then TARGET_OWLBOT="$SOURCE_REPO_NAME/owlbot.py" - python3 "$TRANSFORM_OWLBOT_SCRIPT" "$TARGET_OWLBOT" "$SOURCE_DIR/owlbot.py" + python3 "$TRANSFORM_OWLBOT_SCRIPT" "$TARGET_OWLBOT" "$SOURCE_DIR/$SOURCE_REPO_NAME/owlbot.py" echo "Committing owlbot.py migration..." git add "$TARGET_OWLBOT" @@ -348,10 +499,54 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.12b Align all version markers across the monorepo +echo "Aligning all version markers using apply_versions.sh..." +bash generation/apply_versions.sh versions.txt current + +# 7.12c Sync all owlbot.py formatting +echo "Syncing all owlbot.py formatting..." +bash generation/update_owlbot_postprocessor_config.sh "$SOURCE_REPO_NAME" || true + +git add -u +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): align versions and format owlbot configurations" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + +# 7.8c Exempt module from global integration testing matrix +echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." +sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" +python3 "$UPDATE_CI_FILTERS_SCRIPT" ".github/workflows/ci.yaml" "$SOURCE_REPO_NAME" +python3 "$UPDATE_CHANGES_FILTERS_SCRIPT" ".github/workflows/ci.yaml" "$SOURCE_REPO_NAME" + +if [ -n "${PRE_INSTALL_DEPS}" ]; then + echo "Injecting explicit dependencies into always_install_deps list inside .kokoro/common.sh..." + for dep in $(echo "${PRE_INSTALL_DEPS}" | tr ',' ' '); do + sed -i.bak "s|always_install_deps_list=(|always_install_deps_list=(\n '${dep}'|" ".kokoro/common.sh" + done + rm -f .kokoro/common.sh.bak +fi + +echo "Committing common.sh and ci.yaml updates..." +git add .kokoro/common.sh .github/workflows/ci.yaml +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): exempt from global integration testing matrix" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + +# 7.9b Conditionally skip version check if unmanaged dependencies exist +echo "Verifying non-release-please version compliance..." +if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then + echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." + python3 "$UPDATE_LINTER_EXCLUSIONS_SCRIPT" "generation/check_non_release_please_versions.sh" "$SOURCE_REPO_NAME" + + echo "Committing linter adjustment..." + git add generation/check_non_release_please_versions.sh + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): skip version check for $SOURCE_REPO_NAME" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +else + echo "All dependency versions fully managed. No linter adjustments required." +fi + # 7.11 Verify compilation echo "Verifying compilation..." BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh -# (cd "$SOURCE_REPO_NAME" && mvn compile -DskipTests -T 1C) # 7.13 Squash commits if [ "${SQUASH_COMMITS:-false}" = "true" ]; then @@ -376,8 +571,10 @@ if [ "${SQUASH_COMMITS:-false}" = "true" ]; then fi # 8. Cleanup -echo "Cleaning up temporary source clone..." -rm -rf "$SOURCE_DIR" +if [ "${SKIP_SOURCE_UPDATE:-false}" != "true" ]; then + echo "Cleaning up temporary source clone..." + rm -rf "$SOURCE_DIR" +fi echo "Migration complete!" echo "The migrated codebase is available in: $TARGET_DIR" diff --git a/monorepo-migration/migrate_issues.sh b/monorepo-migration/migrate_issues.sh new file mode 100755 index 000000000000..373f0ebe2639 --- /dev/null +++ b/monorepo-migration/migrate_issues.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# migrate_issues.sh - Transfer open GitHub issues from one repo to another using gh CLI. + +set -e + +usage() { + echo "Usage: $0 [--label