scripts/build_pg_ext.py
Rama main · réplica de lectura del repositorio original.
import distutils.sysconfig as sysconfig
import json
import os
import sys
import platform
import requests
"""
Usage: python3 scripts/build_pg_ext.py debug #Debug build
Usage: python3 scripts/build_pg_ext.py dev #Develop build
Usage: python3 scripts/build_pg_ext.py prod #Release build
Usage: python3 scripts/build_pg_ext.py debug --deeplake-shared #Debug build with shared deeplake_api linking
Usage: python3 scripts/build_pg_ext.py debug --deeplake-static #Debug build with static deeplake_api linking (force)
Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17,18 #Build for PostgreSQL 17 and 18
Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17 #Build for PostgreSQL 17 only
Usage: python3 scripts/build_pg_ext.py prod --pg-versions all #Build for all supported PostgreSQL versions
"""
def get_pinned_version():
"""
Read the pinned deeplake API version from DEEPLAKE_API_VERSION file.
"""
# Look for version file in repo root (one level up from scripts/)
script_dir = os.path.dirname(os.path.abspath(__file__))
repo_root = os.path.dirname(script_dir)
version_file = os.path.join(repo_root, "DEEPLAKE_API_VERSION")
if not os.path.exists(version_file):
raise Exception(f"Version file not found: {version_file}")
with open(version_file, 'r') as f:
version = f.read().strip()
if not version:
raise Exception(f"Version file is empty: {version_file}")
return version
def download_api_lib(api_root_dir, overwrite=True):
"""
Download and extract the full deeplake API library including:
- include/ headers
- lib/ libraries
- cmake configurations
- pkg-config files
"""
# create directory for the deeplake api library if it does not exist
os.makedirs(api_root_dir, exist_ok=True)
machine = platform.machine()
# Get pinned version from DEEPLAKE_API_VERSION file
version = get_pinned_version()
print(f"Using pinned deeplake API version: {version}")
# Check if library already exists
lib_dir = os.path.join(api_root_dir, "lib")
version_marker = os.path.join(api_root_dir, ".version")
if os.path.exists(version_marker) and not overwrite:
with open(version_marker, 'r') as f:
existing_version = f.read().strip()
if existing_version == version:
print(f"Library version {version} already exists. Skipping download.")
return
else:
print(f"Found existing version {existing_version}, but pinned version is {version}. Downloading...")
# Construct asset name based on platform
archive_name = f"deeplake-api-{version}-linux-{machine}"
zip_archive_name = f"{archive_name}.zip"
tar_archive_name = f"{archive_name}.tar.gz"
# Construct download URL directly (GitHub releases follow a predictable URL pattern)
asset_url = f"https://github.com/activeloopai/deeplake/releases/download/v{version}/{zip_archive_name}"
print(f"Downloading prebuilt api libraries from {asset_url} ...")
response = requests.get(asset_url)
if response.status_code != 200:
raise Exception(f"Failed to download api libraries from {asset_url}. Status code: {response.status_code}")
with open(f"{zip_archive_name}", "wb") as f:
f.write(response.content)
print("Extracting archives...")
err = os.system(f"unzip -o {zip_archive_name} -d /tmp && rm {zip_archive_name}")
if err:
raise Exception(f"Failed to extract api libraries. Command exited with code {err}.")
err = os.system(f"tar -xzf /tmp/{tar_archive_name} -C /tmp && rm /tmp/{tar_archive_name}")
if err:
raise Exception(f"Failed to extract api libraries. Command exited with code {err}.")
extracted_path = f"/tmp/{archive_name}"
# Remove existing library directory if overwrite is enabled
if overwrite:
print(f"Removing existing library at {api_root_dir}...")
err = os.system(f"rm -rf {api_root_dir}/*")
if err:
raise Exception(f"Failed to remove existing library. Command exited with code {err}.")
# Copy entire library structure (include/, lib/, cmake configs, pkg-config, etc.)
print(f"Installing full library structure to {api_root_dir}...")
err = os.system(f"cp -r {extracted_path}/* {api_root_dir}/ && rm -rf {extracted_path}")
if err:
raise Exception(f"Failed to copy api library structure. Command exited with code {err}.")
# Write version marker
with open(version_marker, 'w') as f:
f.write(version)
print(f"Successfully installed deeplake API library version {version}")
def run(mode: str, incremental: bool, deeplake_link_type: str = None, pg_versions: list = None):
modes = ["debug", "dev", "prod"]
if mode not in modes:
raise Exception(f"Invalid mode - '{mode}'. Possible values - {', '.join(modes)}")
os.chdir("cpp")
# include path to specific verison from manylinux docker container in the path, so vcpkg_find_acquire_package can find it
os.putenv("PATH", os.getenv("PATH")+f":/opt/_internal/cpython-{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}/bin/")
preset = f"--preset=deeplake-pg-{mode}-windows" if platform.system() == "Windows" else f"--preset=deeplake-pg-{mode}"
try:
if not incremental:
# Download DeepLake API library before CMake configuration
# (CMake's find_package needs it to exist during configuration)
download_api_lib(".ext/deeplake_api")
cmake_cmd = (f"cmake "
f"{preset} ")
architectures = os.environ.get("CMAKE_OSX_ARCHITECTURES", "")
if architectures:
cmake_cmd += f"-D CMAKE_OSX_ARCHITECTURES={architectures} "
# Add PostgreSQL version options if specified
if pg_versions is not None:
supported_versions = {17, 18}
# Set all versions to OFF by default
for ver in supported_versions:
if ver in pg_versions:
cmake_cmd += f"-D BUILD_PG_{ver}=ON "
else:
cmake_cmd += f"-D BUILD_PG_{ver}=OFF "
# Add deeplake linking type option if specified
if deeplake_link_type == "shared":
cmake_cmd += "-D USE_DEEPLAKE_SHARED=ON "
elif deeplake_link_type == "static":
cmake_cmd += "-D USE_DEEPLAKE_SHARED=OFF "
# If None, let CMake auto-detect based on file existence
err = os.system(cmake_cmd)
if err:
raise Exception(
f"Cmake command failed with exit code {err}. Full command: `{cmake_cmd}`"
)
make_cmd = f"cmake --build {preset}"
err = os.system(make_cmd)
if err:
raise Exception(f"Command `{make_cmd}` failed with exit code {err}.")
# Install the built extension to PostgreSQL directories
preset_name = f"deeplake-pg-{mode}-windows" if platform.system() == "Windows" else f"deeplake-pg-{mode}"
install_dir = f"../builds/{preset_name}"
install_cmd = f"cmake --install {install_dir}"
print(f"Installing extension to PostgreSQL directories...")
err = os.system(install_cmd)
if err:
raise Exception(f"Command `{install_cmd}` failed with exit code {err}.")
finally:
os.chdir("..")
write_mode(mode)
def read_mode():
try:
with open('.buildinfo') as f:
try:
data = json.load(f)
except Exception as e:
raise Exception("No previous mode found for incremental build. Please run full build.")
if "mode" in data["deeplake-pg"]:
return data["deeplake-pg"]["mode"]
except Exception as e:
raise Exception("No previous mode found for incremental build. Please run full build.")
def write_mode(mode: str):
data = dict()
try:
with open('.buildinfo', 'r') as f:
try:
data = json.load(f)
except Exception as e:
pass
except Exception as e:
pass
if not "deeplake-pg" in data.keys():
data["deeplake-pg"] = dict()
data["deeplake-pg"]["mode"] = mode
with open('.buildinfo', 'w') as f:
f.write(json.dumps(data))
if __name__ == "__main__":
if len(sys.argv) == 1:
run(mode=read_mode(), incremental=True)
else:
mode = sys.argv[1]
deeplake_link_type = None
pg_versions = None
# Parse optional flags
i = 2
while i < len(sys.argv):
arg = sys.argv[i]
if arg == "--deeplake-shared":
deeplake_link_type = "shared"
i += 1
elif arg == "--deeplake-static":
deeplake_link_type = "static"
i += 1
elif arg == "--pg-versions":
if i + 1 >= len(sys.argv):
raise Exception("--pg-versions requires a value (e.g., '17,18' or 'all')")
versions_str = sys.argv[i + 1]
if versions_str == "all":
pg_versions = [17, 18]
else:
try:
pg_versions = [int(v.strip()) for v in versions_str.split(',')]
# Validate versions
supported = {17, 18}
invalid = set(pg_versions) - supported
if invalid:
raise Exception(f"Invalid PostgreSQL versions: {invalid}. Supported: {supported}")
except ValueError:
raise Exception(f"Invalid --pg-versions format: '{versions_str}'. Use comma-separated numbers (e.g., '17,18') or 'all'")
i += 2
else:
raise Exception(f"Invalid option '{arg}'. Use --deeplake-shared, --deeplake-static, or --pg-versions")
run(mode=mode, incremental=False, deeplake_link_type=deeplake_link_type, pg_versions=pg_versions)