Makefile
Rama main · réplica de lectura del repositorio original.
###############################################################################
# Prelude
###############################################################################
# Many targets in this Makefile assume some commands have been run before to
# setup the correct build environment supporting the different languages
# used for Opengrep development:
# - for OCaml: 'ocamlc' and 'ocamlopt' (currently 5.5.1), 'dune', 'opam'
# - for C: 'gcc', 'ld', 'pkgconfig', but also some C libs like PCRE, gmp
#
# You will also need obviously 'make', but also 'git', and many other
# common dev tools (e.g., 'docker', 'bash').
#
# Once this basic building/development environment has been setup
# (see the different 'install-deps-XXX-yyy' targets later to assist you),
# you can then use:
#
# $ make install-deps
#
# to install the dependencies proper to opengrep (e.g., the necessary OPAM
# packages used by the opengrep engine).
#
# Then to compile opengrep simply type:
#
# $ make all
#
# See INSTALL.md for more information
# Most of the targets in this Makefile should work equally under
# Linux (Alpine, Ubuntu, Arch), macOS (x86 and arm64), and Windows (WSL, Cygwin).
# The main exceptions are the install-deps-XXX-yyy targets below.
# If you really have to use platform-specific commands or flags, try to use
# macros like the one below to have a portable Makefile.
#
# # To select commands with different usage under GNU/Linux and *BSD/Darwin
# ifeq ($(shell uname -s),Linux)
# LINUX = true
# else
# LINUX = false
# endif
# ifeq ($(LINUX),true)
# SED = sed -i -e
# else
# SED = sed -i ''
# endif
BUILD = _build
BUILD_DEFAULT = _build/default
ifeq ($(shell uname -o),Cygwin)
EXE = .exe
endif
###############################################################################
# Build (and clean) targets
###############################################################################
# Set environment variables used by dune files to locate the
# C headers and libraries of the tree-sitter runtime library.
# This file is created by ocaml-tree-sitter-core's configure script.
#
# Because of these required environment variables, we can't call dune directly
# to build opengrep, unless you manually execute first
# `source src/ocaml-tree-sitter-core/tree-sitter-config.sh`
#
# I use '-include' and not 'include' because before 'make setup' this file does
# not exist but we still want 'make setup' to succeed
-include libs/ocaml-tree-sitter-core/tree-sitter-config.mk
# First (and default) target.
.PHONY: default
default: core
# Routine build. It assumes all dependencies and configuration are already in
# place and correct.
.PHONY: all
all:
# OCaml compilation
$(MAKE) core
.PHONY: core
core:
$(MAKE) minimal-build
# Minimal build of the opengrep executable, which also provides the
# low-level engine CLI as 'opengrep --core'.
# If you need other binaries, look at the build-xxx rules below.
.PHONY: minimal-build
minimal-build:
dune build $(BUILD)/install/default/bin/opengrep$(EXE)
# Remove all symbols with GNU strip. It saves 10-25% on the executable
# size and it doesn't seem to reduce the functionality or
# debuggability of OCaml executables.
# See discussion at https://github.com/semgrep/semgrep/pull/9471
chmod +w bin/opengrep$(EXE)
strip bin/opengrep$(EXE)
.PHONY: opengrep-interfile-graph
opengrep-interfile-graph:
dune build $(BUILD)/install/default/bin/opengrep-interfile-graph
.PHONY: opengrep-diff
opengrep-diff:
dune build $(BUILD)/install/default/bin/opengrep-diff
# Run performance benchmarks. Prepares repos/rules then runs hyperfine.
# Results are saved to perf/opengrep-scripts/bench_results/<timestamp>/.
# See perf/opengrep-scripts/README.md for details.
# Usage: make bench [BENCH_ARGS="--dry-run"]
.PHONY: bench
bench: core
cd perf/opengrep-scripts && ./run-benchmarks.sh $(BENCH_ARGS)
.PHONY: build-otarzan
build-otarzan:
dune build $(BUILD)/install/default/bin/otarzan
.PHONY: build-ojsonnet
build-ojsonnet:
dune build $(BUILD)/install/default/bin/ojsonnet
.PHONY: build-pfff
build-pfff:
dune build $(BUILD)/install/default/bin/pfff
# This is an example of how to build one of those parse-xxx ocaml-tree-sitter binaries
.PHONY: build-parse-cairo
build-parse-cairo:
dune build $(BUILD)/install/default/bin/parse-cairo
# Remove from the project tree everything that's not under source control
# and was not created by 'make setup'.
.PHONY: clean
clean:
dune clean
# We still need to keep the nonempty opam files in git for
# 'make setup', so we should only remove the empty opam files.
# This removes the gitignored opam files.
# Disabled: there are no longer any .opam files at the root, they live in
# opam/, and this pathspec matches at any depth rather than just the root.
# git clean -fX *.opam
###############################################################################
# Test target
###############################################################################
# Note that this target is actually not used in CI; it's only for local dev
.PHONY: test
test: core-test
# Experimental - only (re-)run the failed tests
.PHONY: retest
retest:
$(MAKE) build-core-test
./test run --lazy
# Note that this target is actually not used in CI; it's only for local dev
.PHONY: test-all
test-all:
$(MAKE) core-test
#coupling: this is run by .github/workflow/tests.yml
.PHONY: core-test
core-test:
$(MAKE) build-core-test
# The following command ensures that we can call 'test.exe --help'
# from the directory of the checkout
# TODO: this generates weird cmdliner errors in Windows
./test --help 2>&1 >/dev/null
./scripts/run-core-test
# Please keep this standalone target.
# We want to rebuild the tests without re-running all of them.
# This is for working on one or a few specific test cases.
# It rebuilds the test executable which can then be called with
# './test <filter>' where <filter> selects the tests to run.
.PHONY: build-core-test
build-core-test:
dune build $(BUILD_DEFAULT)/src/tests/test.exe
###############################################################################
# External dependencies installation targets
###############################################################################
# **************************************************
# Platform-independent dependencies installation
# **************************************************
# We need to install all the dependencies in a single 'opam install'
# command so as to detect conflicts.
# WEIRD: if you use ./libs/ocaml-tree-sitter-core/ instead of the full
# path, then recent versions of opam crash with a 'git ls-files fatal error'
# about some 'libs/ocaml-tree-sitter-core/../../.git/...' not being a git
# repo.
#
# EXTRA_OPAM_DEPS allows us to add more opam files when building opengrep
# as part of a larger project. Using a single 'opam install' command to
# install all the dependencies allows us to detect version constraints
# incompatibilities.
#
REQUIRED_DEPS = \
./opam/semgrep.opam \
./libs/ocaml-tree-sitter-core/tree-sitter.opam \
./dev/required.opam \
$(EXTRA_OPAM_DEPS)
OPTIONAL_DEPS = $(REQUIRED_DEPS) ./dev/optional.opam
# This target is portable; it only assumes you have 'gcc', 'opam' and
# other build-essential tools and a working OCaml (e.g., ocamlc) switch setup.
# Note that we call opam update below because semgrep.opam may mention
# new packages that are covered yet by our ocaml-layer docker image.
.PHONY: install-deps-for-semgrep-core
install-deps-for-semgrep-core:
opam update -y
# Fetch, build and install the tree-sitter runtime library locally.
cd libs/ocaml-tree-sitter-core \
&& ./configure \
&& ./scripts/install-tree-sitter-lib
make install-opam-deps
# Install OCaml dependencies (globally) from *.opam files.
# This now also installs the dev dependencies. This has the benefit
# of installing all the packages in one shot and detecting possible
# version conflicts.
# OPAMSOLVERTIMEOUT default is 60 but seems not enough
#
# TODO: We use `--assume-depexts` because as of 2024-11-13 brew
# has moved off `pkg-config`. When you install `pkg-config` you
# get `pkgconf` instead, which OCaml doesn't recognize as satisfying
# the dependency though it contains the same elements. This has been
# reported to brew via https://github.com/ocaml/opam-repository/issues/26876.
# We can remove it if that issue is resolved.
# Per the note above install-deps-ALPINE-for-semgrep-core, we may want
# to keep it and add `--no-cache`
install-opam-deps:
opam update -y
OPAMSOLVERTIMEOUT=1200 opam install -y --assume-depexts --deps-only $(REQUIRED_DEPS)
# This will fail if semgrep.opam isn't up-to-date (in git),
# and dune isn't installed yet. You can always install dune with
# 'opam install dune' to get started.
opam/semgrep.opam: dune-project
dune build $@
# Foolproofing
chmod a-w $@
install-deps: install-deps-for-semgrep-core
# **************************************************
# Platform-dependent dependencies installation
# **************************************************
# -------------------------------------------------
# Packages
# -------------------------------------------------
# TODO: opam can in theory handle automatically external dependencies
# via `opam depext ...`, so we should not need for each Linux distro to know
# what is the corresponding package name. This info is actually
# stored in many conf-xxx OPAM packages and since opam 2.2.0 should
# support also Windows!
# So we should generalize our use of WINDOWS_OPAM_DEPEXT_DEPS to all platforms.
# Here is why we need those external packages to compile opengrep:
# - pkgconf (name of pkg-config clone in arch): required by opam/dune
# so it can find the location of the other libs
# - pcre2: regex engine used in the opengrep engine (libpcre2 >= 10.43)
# - gmp: for opengrep and its use of cohttp (LGPL since gmp 6)
# - libev: ?? for Austin
ALPINE_APK_DEPS_CORE=\
pkgconf \
pcre2-dev \
pcre2-static \
gmp-dev \
gmp-static \
libev-dev
# Here is why we need those external packages:
# - pkg-config?
UBUNTU_DEPS=\
pkg-config \
libpcre2-dev \
libgmp-dev \
libev-dev \
#TODO: ARCH_DEPS=??
# NOTE: Additional packages here likely require additional linking flags in
# src/main/flags.sh as part of how we statically link binaries on MacOS.
#
# Here is why we need those external packages:
# - pkg-config?
# - coreutils?
# - gettext?
BREW_DEPS=\
pkg-config \
pcre2 \
gmp \
libev \
coreutils \
gettext
# TODO? why we need those for Windows and not for Linux?
# The opam "depext" are better handled in Linux?
WINDOWS_OPAM_DEPEXT_DEPS=\
conf-pkg-config \
conf-gmp \
conf-libpcre2-8 \
# -------------------------------------------------
# Alpine
# -------------------------------------------------
.PHONY: install-deps-ALPINE
install-deps-ALPINE: install-deps-ALPINE-for-semgrep-core
# Note that we're not using --no-cache below otherwise opam does not
# recognize that pkg-config and other packages have been installed
# (See https://github.com/ocaml/opam/issues/5186)
install-deps-ALPINE-for-semgrep-core:
apk add $(ALPINE_APK_DEPS_CORE)
# -------------------------------------------------
# Ubuntu
# -------------------------------------------------
install-deps-UBUNTU-for-semgrep-core:
sudo apt-get update
apt-get install -y $(UBUNTU_DEPS)
# -------------------------------------------------
# macOS (brew)
# -------------------------------------------------
# see also scripts/osx-setup-for-release.sh that adjust those
# external packages to force static-linking
install-deps-MACOS-for-semgrep-core:
brew install $(BREW_DEPS)
# Install dependencies needed for the Homebrew build.
#
# We don't use just 'make install-deps-for-semgrep-core' because Homebrew
# installs its own version of tree-sitter, globally.
.PHONY: homebrew-setup
homebrew-setup:
cd libs/ocaml-tree-sitter-core \
&& ./configure --prefix "$$(brew --prefix tree-sitter)"
# We pass --no-depexts so as to disable the check for pkg-config
# (which is present due to brew dependencies)
# because this check was failing on some platform.
# See details at https://github.com/Homebrew/homebrew-core/pull/82693.
# This workaround may no longer be necessary.
# LIBRARY_PATH is set here so we build lwt w/libev
LIBRARY_PATH="$$(brew --prefix)/lib" opam install -y --deps-only --no-depexts $(REQUIRED_DEPS)
# -------------------------------------------------
# Arch Linux
# -------------------------------------------------
#TODO: pacman -S ...
# -------------------------------------------------
# Nix
# -------------------------------------------------
# See flake.nix top level comments for more information
# Enter development environment with all dependencies installed
#
# The finger stuff here is weird but it's so we can get the user shell and run
# it in the nix shell. I.e. /usr/bin/zsh or /usr/bin/fish
# It's really weird because by default makefile overrides $SHELL so this is the
# only way to get it
shell:
$(eval USER_SHELL := $(shell finger ${USER} | grep 'Shell:*' | cut -f3 -d ":"))
nix develop -c $(USER_SHELL)
# Build targets
# For all the .?submodules=1 we need because nix is weird:
# https://github.com/NixOS/nix/issues/4423#issuecomment-791352686
nix-osemgrep:
nix build ".?submodules=1#osemgrep"
nix-opengrep:
nix build ".?submodules=1#opengrep"
nix-semgrep:
nix build ".?submodules=1#semgrep"
# Build + run tests
nix-check:
nix flake check ".?submodules=1#"
# verbose and sandboxing are disabled to enable networking for tests
nix-check-verbose:
nix flake check -L ".?submodules=1#"
# check flake is valid and not stale
nix-check-flake:
nix run github:DeterminateSystems/flake-checker
# Update flake inputs
nix-update:
nix flake update
# -------------------------------------------------
# Windows (native, via mingw and cygwin)
# -------------------------------------------------
# used in build-test-windows-x86.jsonnet
install-deps-WINDOWS-for-semgrep-core:
# We will use OPAM 2.1+ which has integrated support for depexts.
opam install -y $(WINDOWS_OPAM_DEPEXT_DEPS)
###############################################################################
# Developer targets
###############################################################################
# This is a best effort to install some external dependencies.
# As a developer you should not run frequently 'make setup', only when
# important dependencies change.
.PHONY: setup
setup: opam/semgrep.opam
./scripts/check-bash-version
$(MAKE) install-deps-for-semgrep-core
# Install optional development dependencies in addition to build dependencies.
.PHONY: dev-setup
dev-setup:
$(MAKE) setup
opam install -y --deps-only $(OPTIONAL_DEPS)
# Update and rebuild everything within the project.
.PHONY: rebuild
rebuild:
git submodule update --init
-$(MAKE) clean
$(MAKE) build
# Same as 'make clean' but may remove additional files, such as external
# libraries installed locally.
#
# Specifically, this removes all files that are git-ignored. New source files
# are preserved, so this command is considered safe.
#
.PHONY: gitclean
gitclean:
git clean -dfX
git submodule foreach --recursive git clean -dfX
# Prepare a release branch.
# This is mainly called by .github/workflows/start-release.yml
# It is safe to run it multiple times.
.PHONY: release
release:
./scripts/release/bump
# Run utop with all the opengrep libraries loaded.
.PHONY: utop
utop:
dune utop
# This is for tools/hello_script.ml so it can leverage the semgrep libs
# (e.g., commons) by installing them in ~/.opam/.../
.PHONY: install-semgrep-libs
install-semgrep-libs: opam/semgrep.opam
dune build
dune install
.PHONY: dump
dump:
$(BUILD_DEFAULT)/tests/test.bc -dump_ast tests/lint/stupid.py
# for ocamldebug
core-bc:
dune build $(BUILD)/install/default/bin/opengrep.bc
test-bc:
dune build $(BUILD_DEFAULT)/src/tests/test.bc
# The bytecode version of opengrep needs dlls for tree-sitter
# stubs installed into ~/.opam/<switch>/lib/stublibs to be able to run.
install-deps-for-semgrep-core-bc: install-deps-for-semgrep-core
dune build @install # Generate the treesitter stubs for below
dune install # Needed to install treesitter_<lang> stubs for use by bytecode
# Run matching performance tests
.PHONY: perf-matching
perf-matching:
@echo "--- default settings ---"
cd ./perf/perf-matching && ./run-perf-suite
@echo "--- no caching ---"
cd ./perf/perf-matching && ./run-perf-suite --no-cache
@echo "--- maximum caching ---"
cd ./perf/perf-matching && ./run-perf-suite --max-cache
# Dogfood opengrep on itself
SEMGREP_ARGS=--config semgrep.jsonnet --error --strict --exclude tests
# you can add --verbose for debugging
.PHONY: check
check:
./bin/opengrep$(EXE) $(SEMGREP_ARGS)
check_for_emacs:
./bin/opengrep$(EXE) $(SEMGREP_ARGS) --emacs --quiet
###############################################################################
# Martin's targets
###############################################################################
# Build executables and place them where opengrep expects them.
#
# Usage:
# $ make dev
# $ ./bin/opengrep ...
.PHONY: dev
dev:
$(MAKE) core