Use .venv dir for as Python venv and cache (#7485)

* Use .venv as Python venv dir

* Update refs to Python venv dir and use action to cache and setup

* Add missing shell: bash

* Source for Python deps

* Exclude .venv from lint

* Update ChangeLog

* Add cache-key arg for init-python

* Add missing "

* Use workflow specific Python cache names

* Fixed cache key for Linux

* Use bash if to make output clearer in case of skipping.

* Clearer debug output

* Add check for cache key

* Add missing shell

* Add SonarCloud and Valgrind venv cache

* Fixed typo

.clang-format

@@ -1,7 +1,7 @@
 # Important: Use the same version of clang-format as our linter, which is:
 #
 # $ ./scripts/install_deps.py --only-python
-# $ ./build/python/bin/clang-format --version
+# $ ./.venv/bin/clang-format --version
 #
 # Warning: If you use a different version, the formatting will be different.
 #
@@ -10,7 +10,7 @@
 
 BasedOnStyle: LLVM
 
-# Turn off LLVM default alignment of params with the opening bracket, 
+# Turn off LLVM default alignment of params with the opening bracket,
 # which can be less readable in some cases in our code base.
 #
 # Using `AlwaysBreak` will result in:

.github/actions/init-python/action.yml

@@ -0,0 +1,50 @@
+name: "Setup Python venv"
+description: "Creates and caches a Python virtual environment"
+
+inputs:
+  cache:
+    description: "Cache Python venv"
+    default: true
+
+  setup:
+    description: "Setup Python venv"
+    default: true
+
+  python-bin:
+    description: "Python binary to use"
+    default: "python3"
+
+  cache-key:
+    description: "Cache key (note: hash is appended)"
+    required: true
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Check cache key
+      if: ${{ inputs.cache }}
+      run: |
+        if [ -z "${{ inputs.cache-key }}" ]; then
+          echo "Cache key is required"
+          exit 1
+        fi
+      shell: bash
+
+    - name: Cache Python venv
+      if: ${{ inputs.cache }}
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: python_venv-${{ inputs.cache-key }}-${{ hashFiles('scripts/pyproject.toml') }}
+
+    # Use bash if to make output clearer in case of skipping.
+    - name: Setup Python venv
+      run: |
+        if [ "${{ inputs.setup }}" = "true" ]; then
+          echo "Setting up Python venv"
+          ${{ inputs.python-bin }} -m venv .venv
+        else
+          echo "Skipping Python venv setup"
+        fi
+      shell: bash

.github/workflows/ci.yml

@@ -71,11 +71,12 @@ jobs:
           key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }}
           restore-keys: vcpkg-${{ runner.os }}
 
-      - name: Cache Python env
+      # Should only restore the .venv directory from cache.
-        uses: actions/cache@v4
+      - name: Init Python venv
+        uses: ./.github/actions/init-python
         with:
-          path: build/python
+          cache-key: ci-${{ matrix.target.name }}
-          key: python_env-${{ runner.os }}-${{ hashFiles('scripts/pyproject.toml') }}
+          setup: false
 
       - name: Cache deps dir
         uses: actions/cache@v4
@@ -163,6 +164,13 @@ jobs:
       - name: Get version
         uses: ./.github/actions/get-version
 
+      # Should only restore the .venv directory from cache.
+      - name: Setup Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: ci-${{ matrix.target.name }}
+          setup: false
+
       - name: Cache deps dir
         uses: actions/cache@v4
         with:
@@ -281,6 +289,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Should only restore the .venv directory from cache.
+      - name: Setup Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: ci-${{ matrix.distro.name }}
+          setup: false
+
       - name: Get version
         uses: ./.github/actions/get-version
 

.github/workflows/codeql-analysis.yml

@@ -32,6 +32,13 @@ jobs:
       - name: Config Git safe dir
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
+      # Should only restore the .venv directory from cache.
+      - name: Init Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: "codeql"
+          setup: false
+
       - name: Install dependencies
         run: ./scripts/install_deps.py
         env:

.github/workflows/issue-check-stale.yml

@@ -1,4 +1,4 @@
-name: "Close stale issues and PRs"
+name: Close stale issues and PRs
 on:
   schedule:
     - cron: "0 5 * * *"

.github/workflows/issue-check-support.yml

@@ -12,14 +12,19 @@ on:
         required: true
 
 jobs:
-  check-issue-tech-support:
+  issue-check-support:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Check if issue requires tech support
+      - name: Setup Python venv
+        uses: ./.github/actions/
+        with:
+          cache-key: "ubuntu-latest"
+
+      - name: Check issue
         id: issue-check
         run: ./scripts/github.py --issue-check-tech-support
         env:

.github/workflows/lint-source-code.yml

@@ -20,10 +20,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Setup Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: "lint-source-code"
+
       - name: Install dependencies
         run: |
-          python3 -m venv build/python
+          source .venv/bin/activate
-          source build/python/bin/activate
           pip install pyyaml cmake_format clang_format
 
       - name: Linting with CMake formatter

.github/workflows/sonarcloud-analysis.yml

@@ -35,6 +35,13 @@ jobs:
       - name: Config Git safe dir
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
+      # Should only restore the .venv directory from cache.
+      - name: Init Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: "sonarcloud"
+          setup: false
+
       - name: Install dependencies
         run: |
           ./scripts/install_deps.py &&

.github/workflows/valgrind-analysis.yml

@@ -22,6 +22,13 @@ jobs:
       - name: Config Git safe dir
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
+      # Should only restore the .venv directory from cache.
+      - name: Init Python venv
+        uses: ./.github/actions/init-python
+        with:
+          cache-key: "valgrind"
+          setup: false
+
       - name: Install dependencies
         run: |
           ./scripts/install_deps.py &&

.gitignore

@@ -6,9 +6,10 @@
 /vcpkg
 /vcpkg_installed
 /scripts/**/*.pyc
+/.cache
+/.venv
 aqtinstall.log
 Brewfile.lock.json
-/.cache
 
 # typical developer-created files
 synergy-config.toml

ChangeLog

@@ -16,6 +16,7 @@ Enhancements:
 - #7473 Simplify `vcpkg` caching and use system `vcpkg`
 - #7474 FreeBSD GitHub runner with `vmactions/freebsd-vm@v1`
 - #7479 Add `BUILD.md` to get people started
+- #7485 Use `.venv` dir for as Python venv and cache
 
 # 1.15.1
 

cmake/Libraries.cmake

@@ -600,10 +600,11 @@ macro(configure_coverage)
 endmacro()
 
 macro(configure_python)
+  set(python_venv_dir ${CMAKE_SOURCE_DIR}/.venv)
   if(WIN32)
-    set(PYTHON_BIN "${CMAKE_BINARY_DIR}/python/Scripts/python.exe")
+    set(PYTHON_BIN ${python_venv_dir}/Scripts/python.exe)
   else()
-    set(PYTHON_BIN "${CMAKE_BINARY_DIR}/python/bin/python")
+    set(PYTHON_BIN ${python_venv_dir}/bin/python)
   endif()
 endmacro()
 

cspell.json

@@ -7,6 +7,7 @@
     "aqtinstall",
     "Axelson",
     "Breen",
+    "cmakelang",
     "codesign",
     "codesigning",
     "Compat",
@@ -64,6 +65,8 @@
     "unittests",
     "Valgrind",
     "vcpkg",
+    "venv",
+    "vmactions",
     "Volker",
     "whot",
     "winget"

scripts/lib/env.py

@@ -16,7 +16,8 @@
 import os, sys, subprocess
 import lib.cmd_utils as cmd_utils
 
-venv_path = "build/python"
+# The `.venv` dir seems to be most common for virtual environments.
+VENV_DIR = ".venv"
 
 
 def check_module(module):
@@ -105,9 +106,9 @@ def get_env_bool(name, default=False):
 
 def get_python_executable(binary="python"):
     if sys.platform == "win32":
-        return os.path.join(venv_path, "Scripts", binary)
+        return os.path.join(VENV_DIR, "Scripts", binary)
     else:
-        return os.path.join(venv_path, "bin", binary)
+        return os.path.join(VENV_DIR, "bin", binary)
 
 
 def in_venv():
@@ -125,13 +126,13 @@ def ensure_in_venv(script_file, auto_create=False):
     import venv
 
     if not in_venv():
-        if not os.path.exists(venv_path):
+        if not os.path.exists(VENV_DIR):
             if not auto_create:
                 print("Hint: Run the `install_deps.py` script first.")
-                raise RuntimeError(f"Virtual environment not found at: {venv_path}")
+                raise RuntimeError(f"Virtual environment not found at: {VENV_DIR}")
 
-            print(f"Creating virtual environment at {venv_path}")
+            print(f"Creating virtual environment at {VENV_DIR}")
-            venv.create(venv_path, with_pip=True)
+            venv.create(VENV_DIR, with_pip=True)
 
         script_file_abs = os.path.abspath(script_file)
         print(f"Using virtual environment for: {script_file_abs}", flush=True)

scripts/lint_cmake.py

@@ -23,12 +23,17 @@ import sys, argparse
 import lib.fs as fs
 from cmakelang.format.__main__ import main as cmake_format_main  # type: ignore
 
-include_files = [
+INCLUDE_FILES = [
     "*.cmake",
     "CMakeLists.txt",
 ]
 
-exclude_dirs = ["subprojects", "build", "deps"]
+EXCLUDE_DIRS = [
+    "build",
+    ".venv",
+    "deps",
+    "subprojects",
+]
 
 
 def main():
@@ -46,7 +51,7 @@ def main():
     args = parser.parse_args()
 
     cmd_args = ["--in-place"] if args.format else ["--check"]
-    files_recursive = fs.find_files(".", include_files, exclude_dirs)
+    files_recursive = fs.find_files(".", INCLUDE_FILES, EXCLUDE_DIRS)
 
     if args.format:
         print("Formatting files with CMake formatter:")