diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 8789531ae077..1d0a7b331591 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -59,7 +59,7 @@ labelPRBasedOnFilePath: - "**/*.Rd" verifyTitles: - titleRegexp: '^(WIP:?\s*)?GH-[0-9]+|^MINOR: ' + titleRegexp: '^GH-[0-9]+|^MINOR: ' alwaysUsePrTitle: true validateEitherPrOrSingleCommitTitle: false statusTitle: "PR Title" @@ -67,8 +67,12 @@ verifyTitles: failureMessage: "Title must start with 'GH-: [Component]' or 'MINOR: [Component]'" insertIssueLinkInPrDescription: - descriptionIssuePlaceholderRegexp: '^\* GitHub Issue: (.*)$' + descriptionIssuePlaceholderRegexp: '^\* GitHub Issue: (?!Refs )(.*)$' matchers: + closedIssue: + targetBranchFilter: '^main$' + titleIssueIdRegexp: '^GH-([0-9]+)' + descriptionIssueLink: 'Closes #${1}' githubIssue: titleIssueIdRegexp: '^(?:WIP:?\s*)?GH-([0-9]+)' descriptionIssueLink: '#${1}' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dd7509e2605a..313d0a6269fc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -34,4 +34,10 @@ In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/de - [ ] AI - [ ] Not reviewed + * GitHub Issue: (filled in automatically from the PR title) diff --git a/.github/workflows/dev_pr_merged.yml b/.github/workflows/dev_pr_merged.yml new file mode 100644 index 000000000000..8d388816b693 --- /dev/null +++ b/.github/workflows/dev_pr_merged.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev PR merged + +on: + push: + branches: + - main + +permissions: + contents: read + issues: write + +jobs: + update: + name: Update issue + if: github.repository == 'apache/arrow' + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + steps: + # GitHub closes the issue of a "GH-XXX: ..." pull request on merge + # because of "* GitHub Issue: Closes #XXX" in the pull request + # description. The following steps set what GitHub doesn't set: + # the assignee and the milestone of the issue. + - name: Get issue + id: issue + env: + MESSAGE: ${{ github.event.head_commit.message }} + run: | + set -euo pipefail + + title=${MESSAGE%%$'\n'*} + if [[ ! "${title}" =~ ^GH-([0-9]+) ]]; then + echo "No GitHub issue in the commit title: ${title}" + exit 0 + fi + issue_number=${BASH_REMATCH[1]} + issue=$(gh api "repos/${GH_REPO}/issues/${issue_number}") + if [ "$(jq 'has("pull_request")' <<< "${issue}")" = "true" ]; then + echo "#${issue_number} is a pull request, not an issue." + exit 0 + fi + # The issue is open if the pull request description didn't + # have "Closes #XXX", for example for a partial fix. + if [ "$(jq -r .state <<< "${issue}")" != "closed" ]; then + echo "::notice::#${issue_number} is still open. It isn't updated." + exit 0 + fi + + # The following steps don't run without these outputs. + { + echo "number=${issue_number}" + echo "n_assignees=$(jq '.assignees | length' <<< "${issue}")" + echo "has_milestone=$(jq '.milestone != null' <<< "${issue}")" + } | tee -a "${GITHUB_OUTPUT}" + # Assign the issue to the commit author, who is the pull request + # author for a squash merge, if it isn't assigned to anyone. + # GitHub ignores this silently if the author can't be assigned. + - name: Assign issue + if: >- + steps.issue.outputs.n_assignees == '0' && + github.event.head_commit.author.username + env: + AUTHOR: ${{ github.event.head_commit.author.username }} + ISSUE_NUMBER: ${{ steps.issue.outputs.number }} + run: | + gh api "repos/${GH_REPO}/issues/${ISSUE_NUMBER}/assignees" \ + -f "assignees[]=${AUTHOR}" > /dev/null + # Keep the milestone of the issue. If the issue doesn't have a + # milestone, use the next major release that doesn't have a + # maintenance branch yet like dev/merge_arrow_pr.py does. + - name: Set milestone + if: steps.issue.outputs.has_milestone == 'false' + env: + ISSUE_NUMBER: ${{ steps.issue.outputs.number }} + run: | + set -euo pipefail + + maintenance_versions=$( + gh api --paginate "repos/${GH_REPO}/git/matching-refs/heads/maint-" \ + --jq '.[].ref | ltrimstr("refs/heads/maint-")') + major_versions=$( + gh api --paginate "repos/${GH_REPO}/milestones?per_page=100" \ + --jq '.[] | select(.title | test("^[0-9]+\\.0\\.0$")) | "\(.title):\(.number)"' | + sort -V) + for major_version in ${major_versions}; do + if ! grep -qxF "${major_version%:*}" <<< "${maintenance_versions}"; then + echo "Milestone: ${major_version%:*}" + gh api -X PATCH "repos/${GH_REPO}/issues/${ISSUE_NUMBER}" \ + -F "milestone=${major_version#*:}" > /dev/null + break + fi + done diff --git a/dev/README.md b/dev/README.md index 275e1ae5c92b..5f2dd8e57506 100644 --- a/dev/README.md +++ b/dev/README.md @@ -33,8 +33,40 @@ committer. ## How to Merge a Pull Request -Please don't merge PRs using the GitHub Web interface. Instead, run -the following command: +Use the "Squash and merge" button of the GitHub Web interface. Before +you merge, check that: + +* The pull request title is what you want as the commit title: + `GH-XXX: [Component] Title` or `MINOR: [Component] Title`. The + "PR Title" status of the Boring Cyborg app + (`.github/boring-cyborg.yml`) fails unless the title starts with + `GH-XXX` or `MINOR: ` but it doesn't block the merge. +* The last line of the pull request description is + `* GitHub Issue: Closes #XXX`. The Boring Cyborg app writes it for a + pull request to `main` and GitHub closes the issue on merge because + of it. If the pull request fixes only a part of the issue, change + the line to `* GitHub Issue: Refs #XXX` before you merge: the issue + stays open. If the line doesn't say `Closes #XXX`, for example + because the app is down, change it or close the issue yourself. +* The pull request description is what you want as the commit + message. Remove leftovers of the pull request template in the + merge dialog if needed. You should remove CI directives such as + `[skip ci]`: GitHub doesn't run the workflow below for a commit + with them. + +After the merge, the "Dev PR merged" GitHub Actions workflow reads +`GH-XXX` from the commit title. If GitHub closed the issue, the +workflow assigns it to the pull request author if it isn't assigned to +anyone and sets its milestone. The existing issue milestone is +retained: set it before merging if the pull request isn't targeted for +the next major release. If the issue doesn't have a milestone, the next +major release is used. This is only done for pull requests to `main`. + +### Merge script + +The merge script is still available for now but using the merge button is encouraged. It additionally adds +`Authored-by:`, `Co-authored-by:` and `Signed-off-by:` trailers to the +commit message. Run the following command: ```bash dev/merge_arrow_pr.sh diff --git a/dev/archery/archery/release/cli.py b/dev/archery/archery/release/cli.py index 58979406c337..f95141bcc037 100644 --- a/dev/archery/archery/release/cli.py +++ b/dev/archery/archery/release/cli.py @@ -18,9 +18,10 @@ import pathlib import click +from git import Repo from ..utils.cli import validate_arrow_sources -from .core import IssueTracker, Release +from .core import IssueTracker, Release, count_committers @click.group('release') @@ -52,6 +53,19 @@ def release_curate(obj, version, minimal): click.echo(curation.render('console')) +@release.command('committers') +@click.argument('previous_version') +@click.argument('version') +@click.pass_obj +def release_committers(obj, previous_version, version): + """Count the pull requests of a release by who merged them.""" + commit_range = f"apache-arrow-{previous_version}..apache-arrow-{version}" + commits = Repo(obj['repo']).iter_commits(commit_range) + committers = count_committers(commits, obj['issue_tracker'].github_repo) + for name, count in committers: + click.echo(f"{count:6d}\t{name}") + + @release.group('changelog') def release_changelog(): """Release changelog.""" diff --git a/dev/archery/archery/release/core.py b/dev/archery/archery/release/core.py index 3ce2110a5506..e0521d8209da 100644 --- a/dev/archery/archery/release/core.py +++ b/dev/archery/archery/release/core.py @@ -242,6 +242,46 @@ def title(self): return self._title +_PULL_REQUEST_NUMBER_REGEX = re.compile(r" \(#(\d+)\)$") + + +def count_committers(commits, github_repo): + """ + Count the pull requests of the commits by who merged them. + + We ask GitHub because commits merged with the merge button don't + have "Signed-off-by:" trailers. + + Returns (name, count) pairs sorted by count (descending) and name. + """ + # Release commits and direct pushes don't have a pull request number. + matches = (_PULL_REQUEST_NUMBER_REGEX.search(commit.summary) + for commit in commits) + numbers = [int(match.group(1)) for match in matches if match] + + counts = defaultdict(int) + # We use GraphQL because it can return many pull requests per + # request. REST needs a request per pull request. + n_per_request = 50 + for i in range(0, len(numbers), n_per_request): + batch = numbers[i:i + n_per_request] + fields = " ".join( + f"pr{number}: pullRequest(number: {number}) " + "{mergedBy {login ... on User {name}}}" + for number in batch) + query = ('query {repository(owner: "apache", name: "arrow") {' + f'{fields}}}}}') + _, result = github_repo.requester.graphql_query(query, {}) + for number in batch: + merger = result["data"]["repository"][f"pr{number}"]["mergedBy"] + if merger is None: + warnings.warn(f"#{number} doesn't have a merger. It isn't " + "counted.") + continue + counts[merger.get("name") or merger["login"]] += 1 + return sorted(counts.items(), key=lambda item: (-item[1], item[0])) + + class Release: def __new__(self, version, repo=None, github_token=None, diff --git a/dev/archery/archery/release/tests/test_release.py b/dev/archery/archery/release/tests/test_release.py index fae2bdcea04a..f77743b9191b 100644 --- a/dev/archery/archery/release/tests/test_release.py +++ b/dev/archery/archery/release/tests/test_release.py @@ -15,11 +15,13 @@ # specific language governing permissions and limitations # under the License. +from types import SimpleNamespace + import pytest from archery.release.core import ( Release, MajorRelease, MinorRelease, PatchRelease, - IssueTracker, Version, Issue, CommitTitle, Commit + IssueTracker, Version, Issue, CommitTitle, Commit, count_committers ) @@ -355,3 +357,57 @@ def test_maintenance_patch_selection(fake_issue_tracker): '9123dadfd123bca7af4eaa9455f5b0d1ca8b929d', ] assert shas_to_pick == expected + + +def test_count_committers(): + commits = [ + SimpleNamespace(summary=summary) for summary in [ + "GH-1: [C++] Title (#11)", + "GH-2: [Python] Title (#12)", + "MINOR: [Release] Update versions for 2.0.0", + "GH-3: [R] Title (#13)", + "GH-4: [Docs] Title (#14)", + "GH-5: [CI] Mention (#10) in the middle", + "GH-6: [Ruby] Title (#15)", + "GH-7: [Go] Title (#16)", + ] + ] + mergers = { + "pr11": {"login": "zoe", "name": "Zoe Z"}, + "pr12": {"login": "bob", "name": "Bob B"}, + "pr13": {"login": "zoe", "name": "Zoe Z"}, + # a user without a name: the login is used + "pr14": {"login": "alice", "name": None}, + # no merger: not counted + "pr15": None, + # not a user, for example a bot: the login is used + "pr16": {"login": "carol"}, + } + queries = [] + + def graphql_query(query, variables): + queries.append(query) + pull_requests = { + alias: {"mergedBy": merger} for alias, merger in mergers.items() + } + return {}, {"data": {"repository": pull_requests}} + + github_repo = SimpleNamespace( + requester=SimpleNamespace(graphql_query=graphql_query)) + + with pytest.warns(UserWarning, match="#15 doesn't have a merger"): + committers = count_committers(commits, github_repo) + + assert committers == [ + ("Zoe Z", 2), + ("Bob B", 1), + ("alice", 1), + ("carol", 1), + ] + assert queries == [ + 'query {repository(owner: "apache", name: "arrow") {' + + " ".join(f"pr{number}: pullRequest(number: {number}) " + "{mergedBy {login ... on User {name}}}" + for number in range(11, 17)) + + '}}' + ] diff --git a/dev/archery/setup.py b/dev/archery/setup.py index 0b2bbfcb89c6..30c064ad951a 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -35,7 +35,7 @@ 'integration': ['cffi', 'numpy'], 'integration-java': ['jpype1'], 'numpydoc': ['numpydoc==1.1.0'], - 'release': ['pygithub<2.9.0', jinja_req, 'semver', 'gitpython'], + 'release': ['pygithub>=2.5.0,<2.9.0', jinja_req, 'semver', 'gitpython'], } extras['bot'] = extras['crossbow'] + ['pygithub<2.9.0'] extras['all'] = list(set(functools.reduce(operator.add, extras.values()))) diff --git a/dev/release/post-04-website.sh b/dev/release/post-04-website.sh index 41a7ee4a04cf..df755ef608c0 100755 --- a/dev/release/post-04-website.sh +++ b/dev/release/post-04-website.sh @@ -35,6 +35,10 @@ if ! archery --help > /dev/null 2>&1; then exit 1 fi +# Without a token archery waits on the rate limit of GitHub for a long +# time instead of failing. +: "${GH_TOKEN:?GH_TOKEN is required to ask GitHub who merged the pull requests}" + previous_version=$1 version=$2 @@ -75,10 +79,11 @@ rough_n_development_months=$(( git_tag=apache-arrow-${version} git_range=apache-arrow-${previous_version}..${git_tag} -committers_command_line="git shortlog -sn --group=trailer:signed-off-by ${git_range}" contributors_command_line="git shortlog -sn ${git_range}" -committers=$(${committers_command_line}) +# Count the pull requests of the commits in ${git_range} by who merged +# them. This asks GitHub: it needs GH_TOKEN. +committers=$(archery release committers ${previous_version} ${version}) contributors=$(${contributors_command_line}) n_commits=$(git log --pretty=oneline ${git_range} | wc -l) @@ -151,8 +156,7 @@ cat <> "${announce_file}" The following Apache committers merged contributed patches to the repository. -\`\`\`console -$ ${committers_command_line} +\`\`\`text ANNOUNCE echo "${committers}" >> "${announce_file}" diff --git a/docs/source/developers/continuous_integration/overview.rst b/docs/source/developers/continuous_integration/overview.rst index 4ae0e97297c4..43204a994c9c 100644 --- a/docs/source/developers/continuous_integration/overview.rst +++ b/docs/source/developers/continuous_integration/overview.rst @@ -53,6 +53,7 @@ The ``.yml`` files in ``.github/workflows`` are workflows which are run on GitHu - ``@github-actions crossbow submit ...`` - runs the specified Crossbow command - ``@github-actions rebase`` - rebases the PR onto the main branch - ``dev.yml`` - runs any time there is activity on a PR, or a PR is merged; it runs the linter and tests that the PR can be merged +- ``dev_pr_merged.yml`` - runs any time a PR is merged to the main branch; assigns the GitHub issue that the PR closed to the PR author if needed and sets its milestone PR housekeeping (component labels, PR title validation and the GitHub issue link in the PR description) is handled by the `Boring Cyborg `_ GitHub App, configured in ``.github/boring-cyborg.yml``. diff --git a/docs/source/developers/overview.rst b/docs/source/developers/overview.rst index 3b8775023236..fe5447c31382 100644 --- a/docs/source/developers/overview.rst +++ b/docs/source/developers/overview.rst @@ -133,8 +133,13 @@ in the end. To make the review process smooth for everyone, try to When tests are passing and the pull request has been approved by the interested parties, a `committer `_ -will merge the pull request. This is done with a -**command-line utility that does a squash merge**. +will merge the pull request. This is done with a **squash merge**. + +The ``* GitHub Issue:`` line at the end of the pull request description is +filled in automatically from the pull request title. For a pull request to +``main`` it says ``Closes #XXX`` and GitHub closes the issue when the pull +request is merged. If your pull request fixes only a part of the issue, change +the line to ``* GitHub Issue: Refs #XXX`` so that the issue stays open. .. dropdown:: Details on squash merge :animate: fade-in-slide-down @@ -147,9 +152,8 @@ will merge the pull request. This is done with a cherry-pick individual patches onto a maintenance branch. Your pull request will appear in the GitHub interface to have been "merged". - In the commit message of that commit, the merge tool adds the pull request - description, a link back to the pull request, and attribution to the contributor - and any co-authors. + The commit message includes the pull request description, a link to the + pull request, and attribution to the contributor and any co-authors. .. _ai-generated-code: diff --git a/docs/source/developers/release.rst b/docs/source/developers/release.rst index fbce61ae9d17..537beffb91e5 100644 --- a/docs/source/developers/release.rst +++ b/docs/source/developers/release.rst @@ -105,7 +105,7 @@ assigned. archery release curate Ensure a major version milestone for a follow up release is created on GitHub. This will -automatically be used by our merge script as the new version for issues closed when +automatically be used on merging pull requests as the new version for issues closed when the maintenance branch is created. Patch Releases @@ -400,7 +400,8 @@ Be sure to go through on the following checklist: :class-title: sd-fs-5 :class-container: sd-shadow-md - Add a release note for the new version to our website and update the latest release information: + Add a release note for the new version to our website and update the latest release information. + The script needs a GitHub token in ``GH_TOKEN`` to ask GitHub who merged the pull requests of the release: .. code-block:: Bash