Uses/cargo: Rework git source support based on patch-in-config sections

Git sources from `Cargo.lock` are added to `CARGO_CRATES` through
the normal mechanism of `make cargo-crates` by the porter.  They
are used to populate `MASTER_SITES`, `DISTFILES` with static
git-archive(1) tarballs a la `USE_GITHUB`, `USE_GITLAB`.  In the
configure phase we generate `[patch]` sections in the config file
which will cause `cargo update` to auto-update `Cargo.lock` to point
to the appropriate extraction directories.

Normally `cargo update` would connect to the network to update all
Git sources but since rust-1.55.0 our cargo has been patched to
skip this when `CARGO_FREEBSD_PORTS_SKIP_GIT_UPDATE` is set in the
environment.

This replaces the old `CARGO_USE_GITHUB`, `CARGO_USE_GITLAB` hacks
where this was done by editing all `Cargo.toml` with sed(1) calls.

Additionally, we try to automatically infer the individiual crate
sub-directories inside the Git sources based on `package.name` in
`Cargo.toml` to remove the need for `CARGO_GIT_SUBDIR`.

USES=cargo also now sets `WRKSRC_crate_$name` for each crate to
point to the crate extraction directories.

PR:		256581
Reviewed by:	jbeich

4 files changed, 272 insertions, 70 deletions

diff --git a/Mk/Scripts/cargo-crates-git-common.awk b/Mk/Scripts/cargo-crates-git-common.awk
new file mode 100644
index 000000000000..c1c5dcc7ce6e
--- /dev/null
+++ b/Mk/Scripts/cargo-crates-git-common.awk
@@ -0,0 +1,105 @@
+# MAINTAINER: rust@FreeBSD.org
+
+BEGIN {
+	# No approval required to add a new gitlab instance here
+	gitlab_hosts["code.videolan.org"] = 1
+	gitlab_hosts["foss.heptapod.net"] = 1
+	gitlab_hosts["framagit.org"] = 1
+	gitlab_hosts["gitlab.com"] = 1
+	gitlab_hosts["gitlab.common-lisp.net"] = 1
+	gitlab_hosts["gitlab.cs.fau.de"] = 1
+	gitlab_hosts["gitlab.dune-project.org"] = 1
+	gitlab_hosts["gitlab.freedesktop.org"] = 1
+	gitlab_hosts["gitlab.gnome.org"] = 1
+	gitlab_hosts["gitlab.howett.net"] = 1
+	gitlab_hosts["gitlab.inria.fr"] = 1
+	gitlab_hosts["gitlab.isc.org"] = 1
+	gitlab_hosts["gitlab.linphone.org"] = 1
+	gitlab_hosts["gitlab.mathematik.uni-stuttgart.de"] = 1
+	gitlab_hosts["gitlab.mn.tu-dresden.de"] = 1
+	gitlab_hosts["gitlab.mpcdf.mpg.de"] = 1
+	gitlab_hosts["gitlab.redox-os.org"] = 1
+	gitlab_hosts["gitlab.torproject.org"] = 1
+	gitlab_hosts["gitlab.xfce.org"] = 1
+	gitlab_hosts["invent.kde.org"] = 1
+	gitlab_hosts["salsa.debian.org"] = 1
+	gitlab_hosts["source.puri.sm"] = 1
+}
+
+function warn(fmt, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z) {
+	printf("WARNING: " fmt "\n", a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z) >"/dev/stderr"
+}
+
+function commit_from_git_url(url) {
+	if (url["query", "tag"]) {
+		return url["query", "tag"]
+	} else {
+		return url["fragment"]
+	}
+}
+
+function split_git_url(info, git_url,		url, path, account, project, commit, i, dir_ver, host) {
+	delete info
+	split_url(url, git_url)
+	url["scheme"] = tolower(url["scheme"])
+	url["host"] = tolower(url["host"])
+	if (url["scheme"] ~ /^git(\+https?)?$/) {
+		if (url["host"] == "github.com") {
+			split(url["path"], path, "/")
+			account = path[2]
+			project = path[3]
+			sub(/\.[gG][iI][tT]$/, "", project)
+			commit = commit_from_git_url(url)
+
+			delete url
+			url["scheme"] = "https"
+			url["host"] = "codeload.github.com"
+			url["path"] = sprintf("/%s/%s/tar.gz/%s", account, project, commit)
+			url["query"] = "dummy"
+			url["query", "dummy"] = "/"
+			info["site"] = join_url(url)
+
+			info["filename"] = sprintf("%s-%s-%s_GH0.tar.gz", account, project, commit)
+
+			# Per bsd.sites.mk:
+			# "GitHub silently converts tags starting with v to not have v in the filename
+			# and extraction directory.  It also replaces + with -."
+			dir_ver = commit
+			sub(/^[vV]/, "", dir_ver)
+			gsub(/\+/, "-", dir_ver)
+			gsub(/--/, "-", dir_ver)
+			info["dir"] = sprintf("%s-%s", project, dir_ver)
+
+			return 1
+		} else if (gitlab_hosts[url["host"]]) {
+			split(url["path"], path, "/")
+			account = path[2]
+			for (i = 3; i < length(path); i++) {
+				account = account "/" path[i]
+			}
+			project = path[i]
+			sub(/\.[gG][iI][tT]$/, "", project)
+			commit = commit_from_git_url(url)
+
+			host = url["host"]
+			delete url
+			url["scheme"] = "https"
+			url["host"] = host
+			url["path"] = sprintf("/%s/%s/-/archive/%s.tar.gz", account, project, commit)
+			url["query"] = "dummy"
+			url["query", "dummy"] = "/"
+			info["site"] = join_url(url)
+
+			gsub(/\//, "-", account)
+			info["filename"] = sprintf("%s-%s-%s_GL0.tar.gz", account, project, commit)
+
+			info["dir"] = sprintf("%s-%s", project, commit)
+
+			return 1
+		}
+	}
+
+	warn("Do not know how to handle %s", git_url)
+	warn("If it points to a GitLab instance try adding the hostname to gitlab_hosts[] at the top of cargo-crates-git-common.awk")
+	return 0
+}
diff --git a/Mk/Scripts/cargo-crates-git-configure.awk b/Mk/Scripts/cargo-crates-git-configure.awk
new file mode 100644
index 000000000000..ade8718e44f6
--- /dev/null
+++ b/Mk/Scripts/cargo-crates-git-configure.awk
@@ -0,0 +1,137 @@
+# MAINTAINER: rust@FreeBSD.org
+
+function parse_sources(		sources_len, raw_sources, i, j, k, original_crate_source, url, crate_source, crate_names, crate_name) {
+	sources_len = 0
+	split(GIT_SOURCES, raw_sources)
+	for (i = 1; i <= length(raw_sources); i++) {
+		j = index(raw_sources[i], "@")
+		if (j == 0) {
+			warn("invalid source: %s\n", raw_sources[i])
+			continue
+		}
+		original_crate_source = substr(raw_sources[i], j + 1)
+		split_url(url, original_crate_source)
+		sub(/^git\+/, "", url["scheme"])
+		delete url["fragment"]
+		delete url["query"]
+		#sub(/\.git$/, "", url["path"])
+		crate_source = join_url(url)
+
+		split(substr(raw_sources[i], 1, j - 1), crate_names, ",")
+		for (k = 1; k <= length(crate_names); k++) {
+			crate_name = crate_names[k]
+			if (!source_crates[crate_source]) {
+				sources[++sources_len] = crate_source
+			}
+			source_crates[crate_source] = source_crates[crate_source] " " crate_name
+			original_crate_sources[crate_source, crate_name] = original_crate_source
+		}
+	}
+}
+
+function get_source_dir(crate_source, crate_name,		git_info, path, in_package, pattern, cmd, cargotoml, line) {
+	if (!split_git_url(git_info, original_crate_sources[crate_source, crate_name])) {
+		exit 1
+	}
+	path = WRKDIR "/" git_info["dir"]
+	# Try to find the first Cargo.toml that defines our crate
+	# We are looking for
+	# [package]
+	# name = "$crate_name"
+	in_package = 0
+	pattern = sprintf("^[ \t]*name[ \t]*=[ \t]*['\"]%s['\"]", crate_name)
+	cmd = FIND " " path " -name Cargo.toml -type f"
+	while ((cmd | getline cargotoml) > 0) {
+		while (getline line <cargotoml) {
+			if (in_package && line ~ pattern) {
+				path = cargotoml
+				sub(/\/Cargo\.toml$/, "", path)
+				close(cmd)
+				close(cargotoml)
+				return path
+			} else if (line ~ /^[ \t]*\[[ \t]*package[ \t]*\][ \t]*$/) {
+				in_package = 1
+			} else if (line ~ /^[ \t]*\[/) {
+				in_package = 0
+			}
+		}
+		close(cargotoml)
+	}
+	close(cmd)
+
+	return path
+}
+
+function find_replaced_crates(input, output,		in_patch_crates_io, line, cols) {
+	delete replaced_crates
+# When Cargo.toml has constructs like this (e.g., www/miniserve, x11/wezterm )
+# [patch.crates-io]
+# mime_guess = { git = "https://github.com/svenstaro/mime_guess.git" }
+# cargo fails to find the replacements we setup.  Check for this
+# and replace with our own [patch.crates-io] section.
+# Note that we need to filter out the original patch section.
+	in_patch_crates_io = 0
+	while (getline line <input) {
+		if (in_patch_crates_io) {
+			if (line ~ /[ \t]*git[ \t]*=/ && line !~ /^[ \t]*#/) {
+				split(line, cols)
+				replaced_crates[cols[1]] = 1
+				print "# " line >output
+				continue
+			}
+		} else if (line ~ /^[ \t]*\[[ \t]*patch\.crates-io[ \t]*\][ \t]*$/) {
+			in_patch_crates_io = 1
+		} else if (line ~ /^[ \t]*\[/) {
+			in_patch_crates_io = 0
+		}
+		print line >output
+	}
+	close(input)
+	close(output)
+}
+
+function add_crates_io_patches(		header_printed, cmd, cargotoml, source, crates) {
+	header_printed = 0
+# --exclude-dir not supported on FreeBSD < 13
+#	cmd = GREP " --include='*/Cargo.toml' --exclude-dir='" CARGO_VENDOR_DIR "' -Flr 'patch.crates-io' " WRKSRC
+	cmd = FIND " " WRKSRC " -name Cargo.toml -not -path '" CARGO_VENDOR_DIR "/*' -exec " GREP " -Flr 'patch.crates-io' {} \\\+"
+	while (cmd | getline cargotoml) {
+		if (0 != system(CP " " cargotoml " " cargotoml ".orig-cargo")) {
+			exit 1
+		}
+		find_replaced_crates(cargotoml ".orig-cargo", cargotoml)
+		if (length(replaced_crates) > 0) {
+			for (i in sources) {
+				source = sources[i]
+				split(source_crates[source], crates)
+				for (j in crates) {
+					if (replaced_crates[crates[j]]) {
+						if (!header_printed) {
+							printf("[patch.crates-io]\n")
+							header_printed = 1
+						}
+						printf("%s = { path = '%s' }\n", crates[j], get_source_dir(source, crates[j]))
+					}
+				}
+			}
+		}
+	}
+	close(cmd)
+}
+
+function add_git_patches(		i, j, source, crates) {
+	for (i = 1; i <= length(sources); i++) {
+		source = sources[i]
+		split(source_crates[source], crates)
+		printf("[patch.'%s']\n", source)
+		for (j = 1; j <= length(crates); j++) {
+			printf("%s = { path = '%s' }\n", crates[j], get_source_dir(source, crates[j]))
+		}
+	}
+}
+
+END {
+	parse_sources()
+	add_crates_io_patches()
+	add_git_patches()
+}
diff --git a/Mk/Scripts/cargo-crates-git-fetch.awk b/Mk/Scripts/cargo-crates-git-fetch.awk
new file mode 100644
index 000000000000..241b6d3b72f2
--- /dev/null
+++ b/Mk/Scripts/cargo-crates-git-fetch.awk
@@ -0,0 +1,20 @@
+# MAINTAINER: rust@FreeBSD.org
+#
+# Return (index, site, filename, wrksrc, crates) 5-tuples from git URL specs in CARGO_CRATES
+
+END {
+	split(GIT_SOURCES, git_sources)
+	for (i = 1; i <= length(git_sources); i++) {
+		git_source = git_sources[i]
+		j = index(git_source, "@")
+		if (j == 0) {
+			warn("invalid source: %s", git_source)
+		} else {
+			crate_source = substr(git_source, j + 1)
+			crates = substr(git_source, 0, j - 1)
+			if (split_git_url(git_info, crate_source)) {
+				printf("%d %s %s %s %s\n", group++, git_info["site"], git_info["filename"], git_info["dir"], crates)
+			}
+		}
+	}
+}
diff --git a/Mk/Scripts/cargo-crates.awk b/Mk/Scripts/cargo-crates.awk
index 56b3beafac11..0cc8fe7a27f9 100644
--- a/Mk/Scripts/cargo-crates.awk
+++ b/Mk/Scripts/cargo-crates.awk
@@ -1,26 +1,15 @@
 # MAINTAINER: rust@FreeBSD.org
 
 BEGIN {
-	gh_tuple_len = 0
-	gl_tuple_len = 0
 	crates_len = 0
-	package_name = "<unknown>"
 	crate_name = "<unknown>"
 	crate_version = "<unknown>"
 	crate_source = "<unknown>"
-
-	gitlab_sites["https://gitlab.com"] = 1
-	gitlab_sites["https://gitlab.freedesktop.org"] = 1
-	gitlab_sites["https://gitlab.gnome.org"] = 1
-	gitlab_sites["https://gitlab.redox-os.org"] = 1
 }
 
 /^name = ".*"/ {
 	crate_name = $3
 	gsub(/"/, "", crate_name)
-
-	package_name = $3
-	gsub("[^a-zA-Z_]", "", package_name)
 }
 
 /^version = ".*"/ {
@@ -38,7 +27,14 @@ BEGIN {
 }
 
 function add_crate() {
-	if (crate_source == "registry+https://github.com/rust-lang/crates.io-index") {
+	if (crate_source ~ /^git\+/) {
+		gsub(/#/, "\\#", crate_source)
+		if (git_crates[crate_source]) {
+			git_crates[crate_source] = git_crates[crate_source] "," crate_name
+		} else {
+			git_crates[crate_source] = crate_name
+		}
+	} else if (crate_source == "registry+https://github.com/rust-lang/crates.io-index") {
 		crates[crates_len++] = sprintf("%s-%s", crate_name, crate_version)
 	}
 	crate_name = "<unknown>"
@@ -46,50 +42,6 @@ function add_crate() {
 	crate_source = "<unknown>"
 }
 
-!gh_tuple_seen[$0] && /^source = "git\+(https|http|git):\/\/.*\/.*#.*"/ {
-	gh_tuple_seen[$0] = 1
-	split_url(url, substr($3, 1 + length("\"git+"), length($3) - 1 - length("\"git+")))
-
-	split(url["path"], path, "/")
-	account = path[2]
-	project = path[3]
-	gsub("\.git$", "", project)
-
-	if (match(url["query"], "^tag=")) {
-		split(url["query"], tag_, "=")
-		tag = tag_[2]
-	} else {
-		tag = url["fragment"]
-	}
-
-	added = 0
-	if (url["host"] == "github.com") {
-		added = 1
-		gh_tuple[gh_tuple_len++] = sprintf(\
-			"%s:%s:%s:%s", account, project, tag, package_name)
-	} else {
-		repo_site = sprintf("%s://%s", url["scheme"], url["host"])
-		for (site in gitlab_sites) {
-			if (repo_site != site) {
-				continue
-			}
-			if (ENVIRON["GL_SITE"] == site) {
-				gl_tuple[gl_tuple_len++] = sprintf(\
-					"%s:%s:%s:%s", account, project, tag, package_name)
-			} else {
-				gl_tuple[gl_tuple_len++] = sprintf(\
-					"%s:%s:%s:%s:%s", site, account, project, tag, package_name)
-			}
-			added = 1
-			break
-		}
-	}
-
-	if (!added) {
-		printf "Warning: Ignoring git source on line %d: %s\n", NR, $3 > "/dev/stderr"
-	}
-}
-
 function print_array(start, arr, arrlen) {
 	end = " \\\n"
 	for (i = 0; i < arrlen; i++) {
@@ -103,20 +55,8 @@ function print_array(start, arr, arrlen) {
 
 END {
 	add_crate()
-
-	if (gh_tuple_len > 0 && ENVIRON["USE_GITHUB"] == "") {
-		printf "USE_GITHUB=\tnodefault\n"
-	}
-	print_array("GH_TUPLE=", gh_tuple, gh_tuple_len)
-	if (gl_tuple_len > 0 && ENVIRON["USE_GITLAB"] == "") {
-		printf "USE_GITLAB=\tnodefault\n"
+	for (crate_source in git_crates) {
+		crates[crates_len++] = git_crates[crate_source] "@" crate_source
 	}
-	print_array("GL_TUPLE=", gl_tuple, gl_tuple_len)
 	print_array("CARGO_CRATES=", crates, crates_len)
-	if (gh_tuple_len > 0) {
-		printf "CARGO_USE_GITHUB=\tyes\n"
-	}
-	if (gl_tuple_len > 0) {
-		printf "CARGO_USE_GITLAB=\tyes\n"
-	}
 }