cluster/tools/rook-s3cmd-config: build using bazel
This turns the existing script into a proper sh_binary, and injects
dependencies (kubectl and jq) as deps into it.
This change also pulls in BUILDfiles for jq, and a dep (oniguruma) into
//third_party, and adds buildable external repositories for them.
The jq/oniguruma BUILDfiles are lifted from
https://github.com/attilaolah/bazel-tools/.
Change-Id: If2e548bd60a8fd34e4f3be767ae59c6b2f2286d9
diff --git a/WORKSPACE b/WORKSPACE
index 268259b..e7f58fc 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -306,6 +306,28 @@
)
go_image_repositories()
+
+# oniguruma, with build from //third_party/oniguruma
+
+http_archive(
+ name = "com_github_kkos_oniguruma",
+ urls = ["https://github.com/kkos/oniguruma/releases/download/v6.9.5_rev1/onig-6.9.5_rev1.tar.gz"],
+ strip_prefix = "onig-6.9.5",
+ sha256 = "d33c849d1672af227944878cefe0a8fcf26fc62bedba32aa517f2f63c314a99e",
+ build_file = "@hscloud//third_party/oniguruma:BUILD.external",
+)
+
+# jq, with build from //third_party/jq
+
+http_archive(
+ name = "com_github_stedolan_jq",
+ urls = ["https://github.com/stedolan/jq/releases/download/jq-1.6/jq-1.6.tar.gz"],
+ strip_prefix = "jq-1.6",
+ sha256 = "5de8c8e29aaa3fb9cc6b47bb27299f271354ebb72514e3accadc7d38b5bbaa72",
+ build_file = "@hscloud//third_party/jq:BUILD.external",
+)
+
+
# Go repositories
go_repository(
diff --git a/cluster/tools/BUILD b/cluster/tools/BUILD
index 8e0900e..d26f668 100644
--- a/cluster/tools/BUILD
+++ b/cluster/tools/BUILD
@@ -37,3 +37,12 @@
data = ["@nixops//:bin", "//tools:secretstore"],
)
+sh_binary(
+ name = "rook-s3cmd-config",
+ srcs = ["rook-s3cmd-config.sh"],
+ data = [
+ "@bazel_tools//tools/bash/runfiles",
+ "@com_github_stedolan_jq//:jq",
+ ":kubectl",
+ ],
+)
diff --git a/cluster/tools/rook-s3cmd-config b/cluster/tools/rook-s3cmd-config
deleted file mode 100755
index 3ec34e5..0000000
--- a/cluster/tools/rook-s3cmd-config
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Generates s3cmd config from rook.io CephObjectStoreUser secrets fetched from
-# Kubernetes apiserver. Accepts extra K8S_INTERNAL=1 environment variable flag
-# that generates config that connects to internal rgw service.
-#
-# Usage:
-# ./rook-s3cmd-config USERNAME STORENAME CLUSTERNAME > config
-# s3cmd -c config --region "STORENAME:default-placement" mb s3://test/
-
-set -e
-
-username="${1:-registry}"
-storename="${2:-waw-hdd-redundant-2-object}"
-clustername="${3:-ceph-waw2}"
-
-secret="$(kubectl get secrets rook-ceph-object-user-$storename-$username -n $clustername -o json)"
-accesskey="$(echo "$secret" | jq -r '.data.AccessKey' | base64 -d)"
-secretkey="$(echo "$secret" | jq -r '.data.SecretKey' | base64 -d)"
-
-if [[ ! -z "$K8S_INTERNAL" ]]; then
- domain="rook-ceph-rgw-$storename.$clustername.svc.cluster.local"
-else
- domain="object.$clustername.hswaw.net"
-fi
-
-cat <<EOF
-[default]
-access_key = $accesskey
-secret_key = $secretkey
-host_base = $domain
-host_bucket = $domain
-EOF
-
-if [[ ! -z "$K8S_INTERNAL" ]]; then
- echo "use_https = False"
-fi
diff --git a/cluster/tools/rook-s3cmd-config.sh b/cluster/tools/rook-s3cmd-config.sh
new file mode 100755
index 0000000..703b597
--- /dev/null
+++ b/cluster/tools/rook-s3cmd-config.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Generates s3cmd config from rook.io CephObjectStoreUser secrets fetched from
+# Kubernetes apiserver. Accepts extra K8S_INTERNAL=1 environment variable flag
+# that generates config that connects to internal rgw service.
+#
+# Usage:
+# bazel run //cluster/tools:rook-s3cmd-config > config
+# s3cmd -c config --region "STORENAME:default-placement" mb s3://test/
+
+set -euo pipefail
+
+# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ if [[ -f "$0.runfiles_manifest" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+ elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+ export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+ elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ export RUNFILES_DIR="$0.runfiles"
+ fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+ source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+ source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+ echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+ exit 1
+fi
+# endpaste
+
+kubectl=$(rlocation "hscloud/cluster/tools/kubectl")
+if [ -z "$kubectl" ]; then
+ echo "Could not find kubectl in runfiles" >&2
+ exit 1
+fi
+
+jq=$(rlocation "com_github_stedolan_jq/jq")
+if [ -z "$jq" ]; then
+ echo "Could not find jq in runfiles" >&2
+ exit 1
+fi
+
+username="${1}"
+storename="${2:-waw-hdd-redundant-3-object}"
+clustername="${3:-ceph-waw3}"
+
+if [ -z "$username" ]; then
+ echo "Usage: $0 <username>" >&2
+ exit 1
+fi
+
+
+secret="$($kubectl get secrets rook-ceph-object-user-$storename-$username -n $clustername -o json)"
+accesskey="$(echo "$secret" | $jq -r '.data.AccessKey' | base64 -d)"
+secretkey="$(echo "$secret" | $jq -r '.data.SecretKey' | base64 -d)"
+
+if [[ ! -z "${K8S_INTERNAL:-}" ]]; then
+ domain="rook-ceph-rgw-$storename.$clustername.svc.cluster.local"
+else
+ domain="object.$clustername.hswaw.net"
+fi
+
+cat <<EOF
+[default]
+access_key = $accesskey
+secret_key = $secretkey
+host_base = $domain
+host_bucket = $domain
+EOF
+
+if [[ ! -z "${K8S_INTERNAL:-}" ]]; then
+ echo "use_https = False"
+fi
diff --git a/third_party/jq/BUILD b/third_party/jq/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/third_party/jq/BUILD
diff --git a/third_party/jq/BUILD.external b/third_party/jq/BUILD.external
new file mode 100644
index 0000000..e26c0fb
--- /dev/null
+++ b/third_party/jq/BUILD.external
@@ -0,0 +1,78 @@
+# Copyright 2019 Google LLC
+# Copyright 2020 Sergiusz 'q3k' Bazanski
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The jq binary.
+cc_binary(
+ name = "jq",
+ srcs = [
+ ":source_files",
+ ],
+ copts = [
+ "-Iexternal/com_github_kkos_oniguruma/src",
+ "-Wno-cpp",
+ "-Wno-unused-function",
+ "-Wno-unused-variable",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ "@com_github_kkos_oniguruma//:oniguruma",
+ ],
+)
+
+# Generate the source files.
+# This will run ./configure and extract a host-dependent confdefs.h file, then
+# prefix each .h acd .c file with '#include "confdefs.h"'.
+genrule(
+ name = "source_files",
+ srcs = glob(["**"]),
+ outs = ["build/" + f for f in glob(
+ [
+ "src/*.h",
+ "src/*.c",
+ ],
+ exclude = [
+ "src/inject_errors.c",
+ ],
+ )] + [
+ # The extracted confdefs.h.
+ "build/src/confdefs.h",
+ # The generated builtin.inc.
+ "build/src/builtin.inc",
+ ],
+ cmd =
+ # Run ./configure && make src/builtin.inc.
+ "( " +
+ " cd external/com_github_stedolan_jq; " +
+ " ./configure > /dev/null 2> /dev/null; " +
+ " make src/builtin.inc > /dev/null; " +
+ "); " +
+ # Extract confdefs.h from config.log.
+ "grep '^/\\* confdefs.h \\*/$$' external/com_github_stedolan_jq/config.log -A1000 " +
+ " | head -n -1 > \"$(@D)\"/build/src/confdefs.h; " +
+ # Prefix each output file with an include of confdefs.h.
+ "OUTS=\"$(OUTS)\"; for FILE in $$OUTS; do " +
+ " touch \"$$FILE\"; " +
+ " BASENAME=\"$$(basename $$FILE)\"; " +
+ " if [ \"$$BASENAME\" != \"confdefs.h\" ]; then " +
+ " echo '#include \"confdefs.h\"' > \"$$FILE\"; " +
+ " cat external/com_github_stedolan_jq/src/\"$$BASENAME\" >> \"$$FILE\"; " +
+ " fi; " +
+ # Replace non-relative references.
+ " sed -e 's|^#include \"src/|#include \"|g' -i \"$$FILE\"; " +
+ "done; " +
+ # Copy builtin.inc and version.h without modificaitons.
+ "cp external/com_github_stedolan_jq/src/builtin.inc \"$(@D)\"/build/src/builtin.inc; ",
+)
+
diff --git a/third_party/jq/README.md b/third_party/jq/README.md
new file mode 100644
index 0000000..223de52
--- /dev/null
+++ b/third_party/jq/README.md
@@ -0,0 +1,6 @@
+third\_party: jq
+================
+
+Upstream: https://github.com/stedolan/jq
+
+Pulled in by WORKSPACE (com\_github\_stedolan\_jq), BUILD.external used as root BUILDfile for external repository.
diff --git a/third_party/oniguruma/BUILD b/third_party/oniguruma/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/third_party/oniguruma/BUILD
diff --git a/third_party/oniguruma/BUILD.external b/third_party/oniguruma/BUILD.external
new file mode 100644
index 0000000..b10f673
--- /dev/null
+++ b/third_party/oniguruma/BUILD.external
@@ -0,0 +1,441 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The Oniguruma library.
+cc_library(
+ name = "oniguruma",
+ hdrs = [
+ "src/oniguruma.h",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":ascii",
+ ":big5",
+ ":cp1251",
+ ":euc_jp",
+ ":euc_kr",
+ ":euc_tw",
+ ":gb18030",
+ ":iso8859",
+ ":koi8",
+ ":onig_init",
+ ":regcomp",
+ ":regenc",
+ ":regerror",
+ ":regexec",
+ ":regext",
+ ":reggnu",
+ ":regparse",
+ ":regposix",
+ ":regsyntax",
+ ":regtrav",
+ ":regversion",
+ ":sjis",
+ ":st",
+ ":unicode",
+ ":utf16",
+ ":utf32",
+ ":utf8",
+ ],
+)
+
+# The mktable binary.
+cc_binary(
+ name = "mktable",
+ srcs = [
+ "src/mktable.c",
+ ],
+ deps = [
+ ":oniguruma",
+ ],
+)
+
+# Other libraries linked in with :oniguruma.
+cc_library(
+ name = "ascii",
+ srcs = [
+ "src/ascii.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "big5",
+ srcs = [
+ "src/big5.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "cp1251",
+ srcs = [
+ "src/cp1251.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "euc_jp",
+ srcs = [
+ "src/euc_jp.c",
+ "src/euc_jp_prop.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "euc_kr",
+ srcs = [
+ "src/euc_kr.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "euc_tw",
+ srcs = [
+ "src/euc_tw.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "gb18030",
+ srcs = [
+ "src/gb18030.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "iso8859",
+ srcs = [
+ "src/iso8859_1.c",
+ "src/iso8859_10.c",
+ "src/iso8859_11.c",
+ "src/iso8859_13.c",
+ "src/iso8859_14.c",
+ "src/iso8859_15.c",
+ "src/iso8859_16.c",
+ "src/iso8859_2.c",
+ "src/iso8859_3.c",
+ "src/iso8859_4.c",
+ "src/iso8859_5.c",
+ "src/iso8859_6.c",
+ "src/iso8859_7.c",
+ "src/iso8859_8.c",
+ "src/iso8859_9.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "koi8",
+ srcs = [
+ "src/koi8.c",
+ "src/koi8_r.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "onig_init",
+ srcs = [
+ "src/onig_init.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regcomp",
+ srcs = [
+ "src/regcomp.c",
+ ],
+ deps = [
+ ":regparse",
+ ],
+ alwayslink = 1,
+ copts = [
+ "-Wno-maybe-uninitialized",
+ ],
+)
+
+cc_library(
+ name = "regenc",
+ srcs = [
+ "src/regenc.c",
+ ],
+ hdrs = [
+ "src/oniguruma.h",
+ "src/regenc.h",
+ "src/regint.h",
+ ],
+ deps = [
+ ":config",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regerror",
+ srcs = [
+ "src/regerror.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regexec",
+ srcs = [
+ "src/regexec.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regext",
+ srcs = [
+ "src/regext.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "reggnu",
+ srcs = [
+ "src/reggnu.c",
+ ],
+ hdrs = [
+ "src/oniggnu.h",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regparse",
+ srcs = [
+ "src/regparse.c",
+ ],
+ hdrs = [
+ "src/regparse.h",
+ ],
+ deps = [
+ ":st",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regposix",
+ srcs = [
+ "src/regposerr.c",
+ "src/regposix.c",
+ ],
+ hdrs = [
+ "src/onigposix.h",
+ ],
+ deps = [
+ ":config",
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regsyntax",
+ srcs = [
+ "src/regsyntax.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regtrav",
+ srcs = [
+ "src/regtrav.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "regversion",
+ srcs = [
+ "src/regversion.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "sjis",
+ srcs = [
+ "src/sjis.c",
+ "src/sjis_prop.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "st",
+ srcs = [
+ "src/st.c",
+ ],
+ hdrs = [
+ "src/st.h",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "unicode",
+ srcs = [
+ "src/unicode.c",
+ "src/unicode_fold1_key.c",
+ "src/unicode_fold2_key.c",
+ "src/unicode_fold3_key.c",
+ "src/unicode_unfold_key.c",
+ ],
+ deps = [
+ ":config",
+ ":data",
+ ":regenc",
+ ":st",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "utf8",
+ srcs = [
+ "src/utf8.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "utf16",
+ srcs = [
+ "src/utf16_be.c",
+ "src/utf16_le.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "utf32",
+ srcs = [
+ "src/utf32_be.c",
+ "src/utf32_le.c",
+ ],
+ deps = [
+ ":regenc",
+ ],
+ alwayslink = 1,
+)
+
+# Generated data files.
+cc_library(
+ name = "data",
+ hdrs = [
+ "src/unicode_egcb_data.c",
+ "src/unicode_fold_data.c",
+ "src/unicode_property_data.c",
+ "src/unicode_property_data_posix.c",
+ "src/unicode_wb_data.c",
+ ],
+)
+
+# CC library containing config.h.
+cc_library(
+ name = "config",
+ hdrs = [
+ ":config_h",
+ ],
+)
+
+genrule(
+ name = "config_h",
+ srcs = glob(["**"]),
+ outs = ["config.h"],
+ cmd =
+ "( " +
+ " cd external/com_github_kkos_oniguruma; " +
+ " ./configure > /dev/null; " +
+ "); " +
+ "cp external/com_github_kkos_oniguruma/src/config.h \"$@\"; ",
+)
+
diff --git a/third_party/oniguruma/README.md b/third_party/oniguruma/README.md
new file mode 100644
index 0000000..eb0d88c
--- /dev/null
+++ b/third_party/oniguruma/README.md
@@ -0,0 +1,6 @@
+third\_party: oniguruma
+=======================
+
+Upstream: https://github.com/kkos/oniguruma
+
+Pulled in by WORKSPACE (com\_github\_kkos\_oniguruma), BUILD.external used as root BUILDfile for external repository.