Release #228
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job | |
| name: Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| duckdb-python-sha: | |
| type: string | |
| description: The commit to build against (defaults to latest commit of current ref) | |
| required: false | |
| duckdb-sha: | |
| type: string | |
| description: The DuckDB submodule commit or ref to build against | |
| required: true | |
| stable-version: | |
| type: string | |
| description: Release a stable version (vX.Y.Z-((rc|post)N)) | |
| required: false | |
| pypi-index: | |
| type: choice | |
| description: Which PyPI to use | |
| required: true | |
| options: | |
| - test | |
| - prod | |
| nightly-stale-after-days: | |
| type: string | |
| description: After how many days should nightlies be considered stale | |
| required: true | |
| default: 3 | |
| store-s3: | |
| type: boolean | |
| description: Also store test packages in S3 (always true for prod) | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build_sdist: | |
| name: Build an sdist and determine versions | |
| uses: ./.github/workflows/packaging_sdist.yml | |
| with: | |
| testsuite: all | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| submodule_pr: | |
| name: Create or update PR to bump submodule to given SHA | |
| needs: build_sdist | |
| uses: ./.github/workflows/submodule_auto_pr.yml | |
| with: | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| secrets: | |
| # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 | |
| DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} | |
| workflow_state: | |
| name: Set state for the release workflow | |
| needs: build_sdist | |
| outputs: | |
| pypi_state: ${{ steps.index_check.outputs.pypi_state }} | |
| ci_env: ${{ steps.ci_env_check.outputs.ci_env }} | |
| s3_url: ${{ steps.s3_check.outputs.s3_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: index_check | |
| name: Check version on PyPI | |
| run: | | |
| set -ex | |
| pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| # install duckdb | |
| curl https://install.duckdb.org | sh | |
| # query pypi | |
| result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs | |
| ---- Output lines | |
| .mode line | |
| ---- Query that fetches the given version's age, if the version already exists | |
| SELECT | |
| today() - (file.value->>'upload_time_iso_8601')::DATE AS age, | |
| FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd | |
| CROSS JOIN json_each(jd.releases) AS rel(key, value) | |
| CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value) | |
| WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}' | |
| LIMIT 1; | |
| EOF | |
| ) | |
| if [ -z "$result" ]; then | |
| pypi_state=VERSION_NOT_FOUND | |
| else | |
| pypi_state=VERSION_FOUND | |
| fi | |
| if [[ -z "${{ inputs.stable-version }}" ]]; then | |
| age=${result#age = } | |
| if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then | |
| echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})" | |
| fi | |
| fi | |
| echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT | |
| - id: ci_env_check | |
| name: Determine CI environment | |
| run: | | |
| set -eu | |
| if [[ test == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-test | |
| elif [[ prod == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-prod${{ inputs.stable-version == '' && '-nightly' || '' }} | |
| else | |
| echo "::error::Invalid value for inputs.pypi-index: ${{ inputs.pypi-index }}" | |
| exit 1 | |
| fi | |
| echo "ci_env=${ci_env}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Using CI environment ${ci_env}" | |
| - id: s3_check | |
| name: Generate S3 upload URL | |
| if: github.repository_owner == 'duckdb' | |
| run: | | |
| set -eu | |
| should_store=${{ (inputs.pypi-index == 'prod' || inputs.store-s3) && '1' || '0' }} | |
| if [[ $should_store == 0 ]]; then | |
| echo "::notice::S3 upload disabled in inputs, not generating S3 URL" | |
| exit 0 | |
| fi | |
| if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then | |
| echo "::warning::S3 upload disabled because package version already uploaded to PyPI" | |
| exit 0 | |
| fi | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| version=${{ needs.build_sdist.outputs.package-version }} | |
| s3_url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/" | |
| echo "::notice::Generated S3 URL: ${s3_url}" | |
| echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT | |
| build_wheels: | |
| name: Build and test releases | |
| needs: workflow_state | |
| if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }} | |
| uses: ./.github/workflows/packaging_wheels.yml | |
| with: | |
| minimal: false | |
| testsuite: all | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| upload_s3: | |
| name: Upload Artifacts to S3 | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, build_wheels, workflow_state] | |
| if: ${{ needs.workflow_state.outputs.s3_url }} | |
| steps: | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: artifacts/ | |
| merge-multiple: true | |
| - name: Upload Artifacts | |
| env: | |
| AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| run: | | |
| aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive | |
| publish_pypi: | |
| name: Publish Artifacts to PyPI | |
| runs-on: ubuntu-latest | |
| needs: [workflow_state, build_sdist, build_wheels] | |
| environment: | |
| name: ${{ needs.workflow_state.outputs.ci_env }} | |
| permissions: | |
| # this is needed for the OIDC flow that is used with trusted publishing on PyPI | |
| id-token: write | |
| steps: | |
| - if: ${{ vars.PYPI_HOST == '' }} | |
| run: | | |
| echo "Error: PYPI_HOST is not set in CI environment '${{ needs.workflow_state.outputs.ci_env }}'" | |
| exit 1 | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: packages/ | |
| merge-multiple: true | |
| - name: Upload artifacts to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/' | |
| packages-dir: packages | |
| verbose: 'true' | |
| cleanup_nightlies: | |
| name: Remove Nightlies from PyPI | |
| needs: [workflow_state, publish_pypi] | |
| if: ${{ inputs.stable-version == '' }} | |
| uses: ./.github/workflows/cleanup_pypi.yml | |
| with: | |
| environment: ${{ needs.workflow_state.outputs.ci_env }} | |
| secrets: | |
| # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| summary: | |
| name: Release summary | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, workflow_state, build_wheels, upload_s3, publish_pypi, cleanup_nightlies] | |
| if: always() | |
| steps: | |
| - run: | | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| pversion=${{ needs.build_sdist.outputs.package-version }} | |
| long_pversion="${pversion} (${sha:0:10})" | |
| pypi_host=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| pypi_duckdb_url=https://${pypi_host}/project/duckdb/${pversion}/ | |
| was_released=${{ needs.publish_pypi.result == 'success' && '1' || '0' }} | |
| if [[ $was_released == 1 ]]; then | |
| echo "## Version ${long_pversion} successfully released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package URL: [${pypi_duckdb_url}](${pypi_duckdb_url})" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "## Version ${long_pversion} was not released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package index state before release: ${{ needs.workflow_state.outputs.pypi_state }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "* Package index: ${pypi_host}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Vendored DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY | |
| echo "* S3 upload status: ${{ needs.upload_s3.result == 'success' && needs.workflow_state.outputs.s3_url || needs.upload_s3.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* CI Environment: ${{ needs.workflow_state.outputs.ci_env }}" >> $GITHUB_STEP_SUMMARY |