diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index eb1b6a71db65b964b95c238905afc6612e054eab..bb350570981d88f64b366f086ed8afc04720d6fc 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -34,10 +34,20 @@ jobs:
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
+      - name: Get changeset status
+        id: get-changeset-status
+        run: |
+          pnpm changeset status --output .changeset/status.json
+          new_version=$(jq -r '.releases[] | select(.name == "llamaindex") | .newVersion' < .changeset/status.json)
+          rm -v .changeset/status.json
+          echo "new-version=${new_version}" >> "$GITHUB_OUTPUT"
+
       - name: Create Release Pull Request or Publish to npm
         id: changesets
         uses: changesets/action@v1
         with:
+          commit: Release ${{ steps.get-changeset-status.outputs.new-version }}
+          title: Release ${{ steps.get-changeset-status.outputs.new-version }}
           # update version PR with the latest changesets
           version: pnpm new-version
           # build package and call changeset publish
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 955d7d99e246211feddceba5a64ad8695eeecb8e..a88543eb1c7e2aac727e627e413df67558be15f3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -92,7 +92,7 @@ Please send a descriptive changeset for each PR.
 ## Publishing (maintainers only)
 
 The [Release Github Action](.github/workflows/release.yml) is automatically generating and updating a
-PR called "Version Packages".
+PR called "Release {version}".
 
 This PR will update the `package.json` and `CHANGELOG.md` files of each package according to
 the current changesets in the [.changeset](.changeset/) folder.