mirror of
https://gitlab.com/buildroot.org/buildroot.git
synced 2026-08-01 21:23:51 -09:00
support/testing: GitTestBase: remove git daemon due to Gitlab-CI security settings
TestGitHash and TestGitRefs are failing on Gitlab-CI since 2025-08-18
(first runtime-test failures report for master) [1] due a new Gitlab-CI
security settings that prevents the any use of localhost (127.0.0.1) on
the Gitlab-CI runner [2]:
"To prevent exploitation of insecure internal web services, all webhook
and integration requests to the following local network addresses are
not allowed:
Private network addresses, including 127.0.0.1, ::1, 0.0.0.0,
10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, and IPv6 site-local
(ffc0::/10) addresses."
Gitlab suggest using a second container as a Services [3] since each
containers have access to one another and can communicate when running
the job. But this is really not practical to run git daemon since the
service is started before the Buildroot git tree is fetched and git
daemon needs git-remote directory content.
See [4]:
"If you have your service that tries to write to CI_PROJECT_DIR
immediately after it's started - it will not work and fail with an
error like '/builds/nknapp/gitlab-ci-test/': No such file or directory
error. Because that directory is most probably not there yet, as the
job step that creates it is yet to be executed!"
Also, using Gitlab-CI services requires to handle GitTestBase setUp
differently on Gitlab-CI than local use of run-tests.
So Gitlab-CI services is not an option.
We could move support/testing/tests/download/git-remote to our Gitlab
group (https://gitlab.com/buildroot.org) but since it's only intended
for testing the git download backend, we don't really want to do that.
So the only remaining choice to clone the repository from the Buildroot
git tree (local machine).
To do so, we have to do the following changes:
- Replace all "git://localhost:$(GITREMOTE_PORT_NUMBER)" by
"$(GITREMOTE_DIR)". GITREMOTE_DIR will contain the absolute path to
tests/download/git-remote directory.
- Force to use git download backend using "SITE_METHOD = git" otherwise
the download infra will fall back using wget download backend.
- Cloning a git repository with submodules using the "file://" protocol
needs to be allowed from git config. It's disabled by default:
"fatal: transport 'file' not allowed"
Use a local gitconfig file generated by GitTestBase before starting
TestGitHash and TestGitRefs with:
[protocol "file"]
allow = always
This is considered safe since we are using a git repository from the
Buildroot git tree.
- Running TestGitHash and TestGitRefs on Gitlab-CI requires additional
git config settings to disable ownership check of the git tree [5].
Thanks to Arnout for advices about GIT_CONFIG_GLOBAL.
Runtime tested:
https://gitlab.com/kubu93/buildroot/-/jobs/11428504450 (TestGitHash)
https://gitlab.com/kubu93/buildroot/-/jobs/11428490035 (TestGitRefs)
[1] https://lore.kernel.org/buildroot/68a3ee65.500a0220.397724.30db@mx.google.com/
[2] https://docs.gitlab.com/security/webhooks/#allow-requests-to-the-local-network-from-webhooks-and-integrations
[3] https://docs.gitlab.com/ci/services/
[4] https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25982#note_2139685797
[5] a016b693f7
Fixes:
https://gitlab.com/buildroot.org/buildroot/-/jobs/11271124174 (TestGitHash)
https://gitlab.com/buildroot.org/buildroot/-/jobs/11271124173 (TestGitRefs)
Cc: Arnout Vandecappelle <arnout@rnout.be>
Signed-off-by: Romain Naour <romain.naour@smile.fr>
Signed-off-by: Julien Olivain <ju.o@free.fr>
This commit is contained in:
Romain Naour
committed by
Julien Olivain
Julien Olivain
parent
5208b7a1bd
commit
a367362af0
@@ -5,6 +5,7 @@
|
||||
################################################################################
|
||||
|
||||
BAD_VERSION = a238b1dfcd825d47d834af3c5223417c8411d90d
|
||||
BAD_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
BAD_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
BAD_SITE_METHOD = git
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
################################################################################
|
||||
|
||||
EXPORT_SUBST_VERSION = 0fdb95cf4f3c5ed4003287649cabb33c5f843e26
|
||||
EXPORT_SUBST_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
EXPORT_SUBST_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
EXPORT_SUBST_SITE_METHOD = git
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
################################################################################
|
||||
|
||||
GOOD_VERSION = a238b1dfcd825d47d834af3c5223417c8411d90d
|
||||
GOOD_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GOOD_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GOOD_SITE_METHOD = git
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
################################################################################
|
||||
|
||||
NOHASH_VERSION = a238b1dfcd825d47d834af3c5223417c8411d90d
|
||||
NOHASH_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
NOHASH_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
NOHASH_SITE_METHOD = git
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_PARTIAL_SHA1_BRANCH_HEAD_VERSION = 68c197d0879d485f4f6c
|
||||
GIT_PARTIAL_SHA1_BRANCH_HEAD_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_PARTIAL_SHA1_BRANCH_HEAD_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_PARTIAL_SHA1_BRANCH_HEAD_SITE_METHOD = git
|
||||
GIT_PARTIAL_SHA1_BRANCH_HEAD_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_BRANCH_VERSION = 317406308d9259e2231b
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_BRANCH_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_BRANCH_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_BRANCH_SITE_METHOD = git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_BRANCH_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_TAG_VERSION = 46bae5b639e5a18e2cc4
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_TAG_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_TAG_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_TAG_SITE_METHOD = git
|
||||
GIT_PARTIAL_SHA1_REACHABLE_BY_TAG_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_PARTIAL_SHA1_TAG_ITSELF_VERSION = 2b0e0d98a49c97da6a61
|
||||
GIT_PARTIAL_SHA1_TAG_ITSELF_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_PARTIAL_SHA1_TAG_ITSELF_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_PARTIAL_SHA1_TAG_ITSELF_SITE_METHOD = git
|
||||
GIT_PARTIAL_SHA1_TAG_ITSELF_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_PARTIAL_SHA1_TAG_POINTS_TO_VERSION = 516c9c5f64ec66534d4d
|
||||
GIT_PARTIAL_SHA1_TAG_POINTS_TO_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_PARTIAL_SHA1_TAG_POINTS_TO_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_PARTIAL_SHA1_TAG_POINTS_TO_SITE_METHOD = git
|
||||
GIT_PARTIAL_SHA1_TAG_POINTS_TO_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SHA1_BRANCH_HEAD_VERSION = 68c197d0879d485f4f6cee85544722b79e68e59f
|
||||
GIT_SHA1_BRANCH_HEAD_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SHA1_BRANCH_HEAD_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SHA1_BRANCH_HEAD_SITE_METHOD = git
|
||||
GIT_SHA1_BRANCH_HEAD_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SHA1_REACHABLE_BY_BRANCH_VERSION = 317406308d9259e2231bd0d6ddad3de3832bce08
|
||||
GIT_SHA1_REACHABLE_BY_BRANCH_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SHA1_REACHABLE_BY_BRANCH_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SHA1_REACHABLE_BY_BRANCH_SITE_METHOD = git
|
||||
GIT_SHA1_REACHABLE_BY_BRANCH_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SHA1_REACHABLE_BY_TAG_VERSION = 46bae5b639e5a18e2cc4dc508f080d566baeff59
|
||||
GIT_SHA1_REACHABLE_BY_TAG_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SHA1_REACHABLE_BY_TAG_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SHA1_REACHABLE_BY_TAG_SITE_METHOD = git
|
||||
GIT_SHA1_REACHABLE_BY_TAG_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SHA1_TAG_ITSELF_VERSION = 2b0e0d98a49c97da6a618ab36337e2058eb733a2
|
||||
GIT_SHA1_TAG_ITSELF_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SHA1_TAG_ITSELF_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SHA1_TAG_ITSELF_SITE_METHOD = git
|
||||
GIT_SHA1_TAG_ITSELF_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SHA1_TAG_POINTS_TO_VERSION = 516c9c5f64ec66534d4d069c2e408d9ae4dce023
|
||||
GIT_SHA1_TAG_POINTS_TO_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SHA1_TAG_POINTS_TO_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SHA1_TAG_POINTS_TO_SITE_METHOD = git
|
||||
GIT_SHA1_TAG_POINTS_TO_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SUBMODULE_DISABLED_VERSION = a9dbc1e23c45e8e1b88c0448763f54d714eb6f8f
|
||||
GIT_SUBMODULE_DISABLED_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SUBMODULE_DISABLED_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SUBMODULE_DISABLED_SITE_METHOD = git
|
||||
GIT_SUBMODULE_DISABLED_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_SUBMODULE_ENABLED_VERSION = a9dbc1e23c45e8e1b88c0448763f54d714eb6f8f
|
||||
GIT_SUBMODULE_ENABLED_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_SUBMODULE_ENABLED_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_SUBMODULE_ENABLED_SITE_METHOD = git
|
||||
GIT_SUBMODULE_ENABLED_GIT_SUBMODULES = YES
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_TAG_VERSION = mytag
|
||||
GIT_TAG_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_TAG_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_TAG_SITE_METHOD = git
|
||||
GIT_TAG_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_WRONG_CONTENT_VERSION = a238b1dfcd825d47d834af3c5223417c8411d90d
|
||||
GIT_WRONG_CONTENT_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_WRONG_CONTENT_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_WRONG_CONTENT_SITE_METHOD = git
|
||||
GIT_WRONG_CONTENT_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
################################################################################
|
||||
|
||||
GIT_WRONG_SHA1_VERSION = 0000000000000000000000000000000000000000
|
||||
GIT_WRONG_SHA1_SITE = git://localhost:$(GITREMOTE_PORT_NUMBER)/repo.git
|
||||
GIT_WRONG_SHA1_SITE = $(GITREMOTE_DIR)/repo.git
|
||||
GIT_WRONG_SHA1_SITE_METHOD = git
|
||||
GIT_WRONG_SHA1_LICENSE_FILES = file
|
||||
|
||||
$(eval $(generic-package))
|
||||
|
||||
51
support/testing/tests/download/gitconfig.py
Normal file
51
support/testing/tests/download/gitconfig.py
Normal file
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import infra
|
||||
|
||||
|
||||
GIT_HOST_DIR = "host"
|
||||
GIT_CONFIG_DIR = os.path.join(GIT_HOST_DIR, "home/br-user")
|
||||
GIT_CONFIG_FILE = os.path.join(GIT_CONFIG_DIR, "gitconfig")
|
||||
|
||||
|
||||
def run_git_config(logfile, cmd):
|
||||
logfile.write(
|
||||
"> running git config with '{}'\n".format(" ".join(cmd)))
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=logfile, stderr=logfile)
|
||||
except FileNotFoundError:
|
||||
logfile.write("> git config failed\n")
|
||||
raise SystemError("git config failed")
|
||||
|
||||
|
||||
def generate_gitconfig(builddir, logtofile, gitremotedir):
|
||||
logfile = infra.open_log_file(builddir, "gitconfig", logtofile)
|
||||
|
||||
# The git repository used by this test is cloned locally from
|
||||
# gitremotedir using the "file://" protocol. Since it contains
|
||||
# several git submodules we need to allow this protocol to be used
|
||||
# with git submodules. Since we don't want to modify the user
|
||||
# (global) gitconfig, we use a local gitconfig file.
|
||||
localgitconfig = os.path.join(builddir, GIT_CONFIG_FILE)
|
||||
|
||||
gitconfigdir = os.path.join(builddir, GIT_CONFIG_DIR)
|
||||
os.makedirs(gitconfigdir, exist_ok=True)
|
||||
|
||||
# We are using the git repository from the Buildroot git tree
|
||||
# (gitremotedir). This repository is safe to use using "file://"
|
||||
# protocol with git submodules.
|
||||
cmd = ["git", "config", "--file", localgitconfig,
|
||||
"--add", "protocol.file.allow", "always"]
|
||||
|
||||
run_git_config(logfile, cmd)
|
||||
|
||||
# Disable ownership check of the git tree for Gitlab-CI
|
||||
# environment.
|
||||
# See: https://gitlab.com/buildroot.org/buildroot/-/commit/a016b693f7830f3c8ae815851d3204b8b6e99821
|
||||
for git_repo in os.scandir(gitremotedir):
|
||||
|
||||
cmd = ["git", "config", "--file", localgitconfig,
|
||||
"--add", "safe.directory", git_repo.path]
|
||||
|
||||
run_git_config(logfile, cmd)
|
||||
@@ -1,47 +0,0 @@
|
||||
# subprocess does not kill the child daemon when a test case fails by raising
|
||||
# an exception. So use pexpect instead.
|
||||
import infra
|
||||
|
||||
import pexpect
|
||||
|
||||
|
||||
GIT_REMOTE_PORT_INITIAL = 9418
|
||||
GIT_REMOTE_PORT_LAST = GIT_REMOTE_PORT_INITIAL + 99
|
||||
|
||||
|
||||
class GitRemote(object):
|
||||
def __init__(self, builddir, serveddir, logtofile):
|
||||
"""
|
||||
Start a local git server.
|
||||
|
||||
In order to support test cases in parallel, select the port the
|
||||
server will listen to in runtime. Since there is no reliable way
|
||||
to allocate the port prior to starting the server (another
|
||||
process in the host machine can use the port between it is
|
||||
selected from a list and it is really allocated to the server)
|
||||
try to start the server in a port and in the case it is already
|
||||
in use, try the next one in the allowed range.
|
||||
"""
|
||||
self.daemon = None
|
||||
self.port = None
|
||||
self.logfile = infra.open_log_file(builddir, "gitremote", logtofile)
|
||||
|
||||
daemon_cmd = ["git", "daemon", "--reuseaddr", "--verbose",
|
||||
"--listen=localhost", "--export-all",
|
||||
"--base-path={}".format(serveddir)]
|
||||
for port in range(GIT_REMOTE_PORT_INITIAL, GIT_REMOTE_PORT_LAST + 1):
|
||||
cmd = daemon_cmd + ["--port={port}".format(port=port)]
|
||||
self.logfile.write("> starting git remote with '{}'\n".format(" ".join(cmd)))
|
||||
self.daemon = pexpect.spawn(cmd[0], cmd[1:], logfile=self.logfile,
|
||||
encoding='utf-8')
|
||||
ret = self.daemon.expect(["Ready to rumble",
|
||||
"Address already in use"])
|
||||
if ret == 0:
|
||||
self.port = port
|
||||
return
|
||||
raise SystemError("Could not find a free port to run git remote")
|
||||
|
||||
def stop(self):
|
||||
if self.daemon is None:
|
||||
return
|
||||
self.daemon.terminate(force=True)
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from tests.download.gitremote import GitRemote
|
||||
import tests.download.gitconfig
|
||||
|
||||
import infra
|
||||
|
||||
@@ -12,29 +12,35 @@ class GitTestBase(infra.basetest.BRConfigTest):
|
||||
BR2_BACKUP_SITE=""
|
||||
"""
|
||||
gitremotedir = infra.filepath("tests/download/git-remote")
|
||||
gitremote = None
|
||||
|
||||
def setUp(self):
|
||||
super(GitTestBase, self).setUp()
|
||||
self.gitremote = GitRemote(self.builddir, self.gitremotedir, self.logtofile)
|
||||
|
||||
self.show_msg("Generating custom git config")
|
||||
tests.download.gitconfig.generate_gitconfig(self.builddir,
|
||||
self.logtofile,
|
||||
self.gitremotedir)
|
||||
|
||||
def tearDown(self):
|
||||
self.show_msg("Cleaning up")
|
||||
if self.gitremote:
|
||||
self.gitremote.stop()
|
||||
if self.b and not self.keepbuilds:
|
||||
self.b.delete()
|
||||
|
||||
def check_hash(self, package):
|
||||
gitconfig = os.path.join(self.builddir,
|
||||
tests.download.gitconfig.GIT_CONFIG_FILE)
|
||||
# store downloaded tarball inside the output dir so the test infra
|
||||
# cleans it up at the end
|
||||
env = {"BR2_DL_DIR": os.path.join(self.builddir, "dl"),
|
||||
"GITREMOTE_PORT_NUMBER": str(self.gitremote.port)}
|
||||
"GITREMOTE_DIR": self.gitremotedir,
|
||||
"GIT_CONFIG_GLOBAL": gitconfig}
|
||||
self.b.build(["{}-dirclean".format(package),
|
||||
"{}-source".format(package)],
|
||||
env)
|
||||
|
||||
def check_download(self, package):
|
||||
gitconfig = os.path.join(self.builddir,
|
||||
tests.download.gitconfig.GIT_CONFIG_FILE)
|
||||
# store downloaded tarball inside the output dir so the test infra
|
||||
# cleans it up at the end
|
||||
dl_dir = os.path.join(self.builddir, "dl")
|
||||
@@ -42,7 +48,8 @@ class GitTestBase(infra.basetest.BRConfigTest):
|
||||
if os.path.exists(dl_dir):
|
||||
shutil.rmtree(dl_dir)
|
||||
env = {"BR2_DL_DIR": dl_dir,
|
||||
"GITREMOTE_PORT_NUMBER": str(self.gitremote.port)}
|
||||
"GITREMOTE_DIR": self.gitremotedir,
|
||||
"GIT_CONFIG_GLOBAL": gitconfig}
|
||||
self.b.build(["{}-dirclean".format(package),
|
||||
"{}-legal-info".format(package)],
|
||||
env)
|
||||
|
||||
Reference in New Issue
Block a user