Initial PoC

11 files changed, 527 insertions, 0 deletions

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..791f88c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,145 @@
+# cloyster: a shell wrapper for machine-oriented shell scripts
+
+_cloyster_ helps you making small, as portable as possible, single-task focused shell scripts
+that are primarily consumed by other scripts or machines.
+
+_cloyster_ provides a portable utilitary functions library to help you be efficient transforming
+and manipulating data.
+
+_cloyster_ scripts are called "commands". A good command does a single thing/action
+(list something, create an interface, bridge an interface, ...).
+
+Being machine-oriented and automation-oriented, commands accepts arguments as JSON, and return
+JSON: this also to easily combine and use commands in commands, filter outputs easily (using `jq`),
+pipe them together...
+
+**Base dependencies**: POSIX shell, jq, jo
+
+## Scripting commands
+
+A command is a POSIX shell script, using `/usr/local/libexec/cloyster` as shebang.
+
+It will run in the _cloyster_ environment and has three outputs:
+
+- the standards outputs, stdout and stderr, which acts as usual: unstructured, line based text, automatically
+  handled by _cloyster_ and returned as `output`,
+- the structured JSON result of your command, set by calling `return_result`, returned as `result`.
+- a structured JSON fatal error, set by calling `return_error`, returned as `error`.
+
+### API
+
+* `return_result JSON|JO` exits successfully with a JSON result
+* `return_error JSON|JO [exit_code=1]` exists with exit_code and a JSON error
+* `args_get [JQ_QUERY=.]` query the arguments using jq
+
+### Utilitary Functions
+
+* `ensure_command_exists BINARY` ensures a binary exists, exits with error if not
+* `os_get [KEY=.|jq] [JQ_QUERY]` operating system version (type, name, pretty_name, version, version_id, supported)
+* `make_random_identifier [SIZE=32]`
+* `date_now_iso_utc`
+* `posix_time_microseconds`
+
+### Arguments/Input
+
+Arguments, or inputs to the command, are also passed in JSON. You have three ways of passing the arguments:
+
+* human friendly: `COMMAND arg1=something ...` (uses `jo`),
+* escaped json string: `COMMAND '{"something": true}'`
+* json wrapped in base64: `COMMAND -b eyJiYW5uZXIiOiJiZWFzdGllIGZvciB0aGUgd2luIn0K`
+
+### Output
+
+```js
+{
+  "error": null /* Error */,
+  "exit_status": 0,
+  "command": "./commands/ifconfig-interfaces-list.sh",
+  "arguments": {},
+  "duration_microseconds": /* microseconds */,
+  "started_at_utc": /* ISO8061Z */,
+  "finished_at_utc": /* ISO8061Z */,
+  "result": /* any */,
+  "output": [
+    // Output Line
+  ]
+}
+```
+
+Output Line:
+
+```js
+{"output": "stdout|stderr", "date":"ISO8061Z", line: "...."}
+```
+
+Error:
+
+```js
+{
+  "type": "execution|command",
+  "error": "error_name",
+  // Error specific field
+}
+```
+
+### Reading script from stdin
+
+You can tell _cloyster_ to execute a script from stdin by setting the `-s` or `-` option to `cloyster`:
+
+```sh
+cat commands/ifconfig-interfaces-list | cloyster -s
+cloyster - <<EOF
+json=$(jls --libxo json | jq -r '.["jail-information"].jail | @json')
+set_result "${json}"
+EOF
+```
+
+### Examples
+
+#### Example: `commands/ifconfig-interfaces-names.sh`
+
+```sh
+#!/usr/local/libexec/cloyster
+echo "stdout something"
+>&2 "stderr something"
+set_result $(ifconfig | grep -E "^[a-z0-9_]+:" | cut -d ':' -f 1 | jq -nRr '[inputs | select(length>0)] | @json')
+```
+
+Run it (piped to `jq` so you beautify JSON output):
+
+```sh
+./commands/ifconfig-interfaces-names.sh | jq
+```
+
+Result:
+
+```json
+{
+  "error": false,
+  "exit_status": 0,
+  "command": "./commands/ifconfig-interfaces-list.sh",
+  "arguments": [],
+  "duration_microseconds": 67,
+  "started_at_utc": "2022-03-22T09:28:51Z",
+  "finished_at_utc": "2022-03-22T09:28:51Z",
+  "result": ["eth0", "eth1", "lo0"],
+  "output": [
+    {"output": "stdout", "date":"iso8601 utc", line: "stdout something"},
+    {"output": "stderr", "date":"iso8601 utc", line: "stderr something"}
+  ]
+}
+```
+
+## Code guide
+
+meh
+
+* We write POSIX Shell compliant code.
+* Use `shellcheck`.
+* Variables:
+  * are considered private/local if they begin by a `_`
+  * are considered public if they are in lowercase
+  * are exported/env if they are in uppercase
+* Cloyster itself must be portable (POSIX Shell) and should be able to work on macOS, Linux, ...
+* 
+
diff --git a/TODO b/TODO
new file mode 100644
index 0000000..3d9d2de
--- /dev/null
+++ b/TODO
@@ -0,0 +1 @@
+- bc isn't on linux
diff --git a/cloyster b/cloyster
new file mode 100755
index 0000000..18f72ca
--- /dev/null
+++ b/cloyster
@@ -0,0 +1,276 @@
+#!/bin/sh
+XTRACE="${XTRACE:-}"
+XTRACE_EXEC="${EXEC_XTRACE:-}"
+DEBUG="${DEBUG:-}"
+hostname="${HOSTNAME:-$(hostname)}"
+_executing=
+_exec_error=
+_script_name=$(basename "$0" | cut -d '.' -f 1)
+
+# is_true VARIABLE
+# return 0 if VARIABLE is "y", "yes", "t", "true", "1". Otherwise returns 1.
+# Example: `if _is_true "${XTRACE}"; then set -x; fi`
+is_true() {
+  _t=1
+  case "$1" in
+    y) _t=0;; yes) _t=0;;
+    t) _t=0;; true) _t=0;;
+    1) _t=0;;
+    *) _t=1;;
+  esac
+  return "${_t}"
+}
+
+if is_true "${XTRACE}"; then set -x; fi
+
+# Exit early if we don't have `jo`.
+if ! which jo > /dev/null 2>&1; then
+  >&2 echo '{"error":{"type":"execution_error","error":"missing_dependency","dependency":"jo"}}'
+  exit 1
+fi
+
+_debug() {
+  _line=$1
+  shift
+  if is_true "${DEBUG}"; then
+    if is_true "${_executing}"; then
+      _date=$(date_now_iso_utc)
+      echo "${_line}" | jo "output=cloyster" "date=${_date}" "line=@-" "${@}" >> "${_cloyster_dest}"
+    else
+      >&2 echo "[${_script_name}:debug] ${_line} $@"
+    fi
+  fi
+}
+
+_execution_error() {
+  _debug "fun=_execution_error() executing=${_executing}"
+  >&2 jo error="$(jo type=execution "$@")"
+}
+
+. ./lib/dependencies.sh
+ensure_command_exists jq
+ensure_command_exists bc
+
+. ./lib/os.sh
+_debug "os=$(os_get jq '. | @json')"
+if ! is_true $(os_get "supported"); then
+  _execution_error error=unsupported_operating_system
+fi
+
+. ./lib/identifiers.sh
+. ./lib/datetime.sh
+
+## getopts
+input_decode=
+output_encode=
+stdin=
+while getopts :Bbs _opt; do
+  case "${_opt}" in
+    b) input_decode="base64";;
+    B) output_encode="base64";;
+    s) stdin=yes;;
+    *) _execution_error error=invalid_option option="${_opt}";;
+  esac
+done
+shift $((OPTIND - 1))
+
+if is_true "${stdin}" || [ "${1}" = "-" ]; then
+  if [ "${1}" = "-" ]; then shift; fi
+  stdin=yes
+  _exec="STDIN"
+else
+  _exec="${1}"
+  shift
+  if [ -z "${_exec}" ]; then
+    _execution_error error=no_script_specified
+  elif [ ! -f "${_exec}" ]; then
+    _execution_error error=missing_script script="${_exec}"
+  elif ! head -n 1 "${_exec}" | grep "/cloyster" > /dev/null 2>&1; then
+    _execution_error error=missing_shebang script="${_exec}"
+  fi
+fi
+
+_args_json=
+if [ ! "${#}" = "0" ]; then
+  _args_json="$(jo -d. $@)"
+  _debug "args_json=$_args_json"
+fi
+
+args_get() {
+  _jqq="${1}"
+  if [ -z "${_jqq}" ]; then
+    _jqq="."
+  fi
+  value="$(echo ${_args_json} | jq -r "${_jqq}")"
+  if [ "${value}" = "null" ]; then value=; fi
+  echo "${value}"
+}
+
+json_quote() {
+  echo "$1" | jq '. | @json' 
+}
+json_unquote() {
+  echo "$1" | jq -r '.'
+}
+
+_trap_exit() {
+  _debug "TRAPPED EXIT >:( $@ executing? ${_executing}"
+  if is_true "${_executing}"; then _exec_post "TRAPEXIT"; fi
+    _debug "TRAPPED EXIT >:( $@ executing? ${_executing}"
+  #rm "${_tmp_base}"* > /dev/null 2>&1 || true
+  ( _json_loggers_stop ) > /dev/null 2>&1
+  _debug "$(jobs)"
+  _debug "bye"
+  if is_true "${_executing}"; then exit 1; else exit 0; fi
+}
+#trap _trap_exit EXIT
+
+_exec_name=$(basename "$_exec")
+_tmp_base="/tmp/${_script_name}@${_exec_name}@$(make_random_identifier 16)"
+_stdout_dest="${_tmp_base}_stdout.log"
+_stderr_dest="${_tmp_base}_stderr.log"
+_cloyster_dest="${_tmp_base}_cloyster.jsonstream"
+_output_dest="${_tmp_base}_output.jsonstream"
+_cntrl_dest="${_tmp_base}_cntrl"
+_tmp_files="${_stdout_dest} ${_stderr_dest} ${_output_dest} ${_cloyster_dest} ${_cntrl_dest}"
+
+_debug "tmp_files: ${_tmp_files}"
+for _f in ${_tmp_files}; do
+  if ! touch "${_f}"; then _execution_error error=tmpfile_creation_failed tmpfile="${_f}"; fi
+done
+
+# _logger_pretty_check NAME ISO806ZDATE LINE...
+_logger_pretty_echo() {
+  _name=$1
+  _date=$2
+  shift 2
+  _line=$@
+  _time=$(echo "$_date" | cut -d 'T' -f 2)  
+  _bold=$(tput md)
+  _format_rst=$(tput me)
+  _color=$(tput AF 3)
+  _time_color=$(tput AF 59)
+  case "${_name}" in
+    stdout) _color=$(tput AF 2);;
+    stderr) _color=$(tput AF 1);;
+    cloyster) _color=$(tput AF 111);;
+  esac
+  >&2 echo -e "${_time_color}${_time} ${_color}${_bold}[${_name}]${_format_rst} ${_line}"
+}
+
+_logger_pids=""
+_json_logger() {
+  _name=$1
+  _log=$2
+  _format=$3
+  _combined_log=${4:-${_output_dest}}
+  _json_logger_job() {
+    _debug "_json_logger ${_name} ${_log}"
+    tail -F "${_log}" | while read -r _line; do
+      _date=$(date_now_iso_utc)
+      case "${_format}" in
+        text)
+              echo "${_line}" | jo "output=${_name}" "date=${_date}" "line=@-" >> "${_combined_log}"
+              _logger_pretty_echo "${_name}" "${_date}" "${_line}"
+              ;;
+        rawjson)
+              echo "${_line}" >> "${_combined_log}"
+              _date=$(echo "${_line}" | jq -r '.date')
+              _logger_pretty_echo "${_name}" "${_date}" "${_line}"
+              ;;
+      esac
+    done
+  }
+  $(_json_logger_job) &
+  _logger_pids="${_logger_pids} $!"
+}
+_json_loggers_stop() {
+    for tailpid in $(ps auxww | grep "tail -F ${_tmp_base}" | grep -v grep |  awk '{ print $2 }'); do
+      kill -s PIPE "${tailpid}" || true
+   done
+  for pid in ${_logger_pids}; do
+      kill "${pid}" > /dev/null 2>&1 || true
+      wait "${pid}" 2>/dev/null || true;
+  done
+}
+_json_logger "cloyster" "${_cloyster_dest}" "rawjson"
+_json_logger "stdout" "${_stdout_dest}" "text"
+_json_logger "stderr" "${_stderr_dest}" "text"
+
+_exec_post() {
+  trap - EXIT
+  _arg=$1
+  _executing=
+  _debug "_exec_post $@"
+  if ! is_true "${XTRACE}" && is_true "${EXEC_XTRACE}"; then set -x; fi
+  end_iso=$(date_now_iso_utc)
+  end_us=$(posix_time_microseconds)
+  duration=$(bc -e "$end_us - $start_us")
+  _json_loggers_stop
+  cat ${_output_dest} | jq -sr '. | @json' > "${_output_dest}.json"
+  eval $(cat $_cntrl_dest)
+  _format_output
+  rm "${_tmp_base}"* || true
+  if [ ! "${_arg}" = "TRAPEXIT" ]; then exit "${_exec_exit_status:-1}"; fi
+}
+_format_output() {
+  jo "hostname=${hostname}" \
+     "command=${_exec}" \
+     "arguments=$(json_unquote ${_args_json:-'{}'})" \
+     "error=${_exec_error:-false}" \
+     "exit_status=${_exec_exit_status}" \
+     "output=:${_output_dest}.json" \
+     "result=${_exec_result}" \
+     "started_at_utc=${start_iso}" \
+     "finished_at_utc=${end_iso}" \
+     "duration_microseconds=${duration}"
+}
+
+_exec_pre() {
+  start_iso=$(date_now_iso_utc)
+  start_us=$(posix_time_microseconds)
+  if ! is_true "${XTRACE}" && is_true "${EXEC_XTRACE}"; then set -x; fi
+  _executing=y
+  trap - EXIT
+}
+
+_exec__trap_exit() {
+  _exit_status=$?
+  _debug "Exec process exited: ${_exit_status}"
+  _exec__put_return "exit_status" "${_exit_status}"
+  if [ ! "${_exit_status}" = "0" ] && ! grep -E "^_exec_error=" ${_cntrl_dest} > /dev/null; then
+    _exec__put_return "error" "$(jo type=command error=exit_status)" "json"
+  fi
+}  
+
+_exec__put_return() {
+  _key=$1
+  _value=$2
+  _format=$3
+  if [ "${_format}" = "json" ]; then
+    _value="$(json_quote ${_value})"
+  fi
+  echo "_exec_${_key}=${_value}" >> ${_cntrl_dest}
+}
+
+_debug "executing: command=${_exec} args=${@}"
+_exec_pre
+if is_true "$stdin"; then
+  eval "$(cat -)" 2> "${_stderr_dest}" > "${_stdout_dest}"
+else
+  SCRIPT="${_exec}"
+  # shellcheck source=/dev/null
+  eval "$(
+     trap _exec__trap_exit EXIT
+     _execution_error() {
+       _exec__put_return "error" "$( jo type=cmd $@)" "json"
+       exit 2
+     }
+     return_result() {
+       _exec__put_return "result" "$1" "json"
+       exit 0
+     }
+     . "${SCRIPT}" 2>"${_stderr_dest}" 1>"${_stdout_dest}"
+  )" || true
+fi
+_exec_post
diff --git a/commands/ifconfig/interfaces-names b/commands/ifconfig/interfaces-names
new file mode 100755
index 0000000..f058404
--- /dev/null
+++ b/commands/ifconfig/interfaces-names
@@ -0,0 +1,3 @@
+#!./cloyster
+
+return_result $(ifconfig | grep -E "^[a-z0-9_]+:" | cut -d ':' -f 1 | jq -nRr '[inputs | select(length>0)] | @json')
diff --git a/commands/jails/list b/commands/jails/list
new file mode 100755
index 0000000..b7e009e
--- /dev/null
+++ b/commands/jails/list
@@ -0,0 +1,6 @@
+#!./cloyster
+
+ensure_command_exists jls
+
+list=$(jls --libxo json | jq -r '.["jail-information"].jail | @json')
+return_result "${list}"
diff --git a/commands/tests/test_ensure_command_exists.sh b/commands/tests/test_ensure_command_exists.sh
new file mode 100755
index 0000000..646d2c1
--- /dev/null
+++ b/commands/tests/test_ensure_command_exists.sh
@@ -0,0 +1,3 @@
+#!./cloyster
+
+ensure_command_exists notgoingtoexist
diff --git a/commands/tests/test_exit.sh b/commands/tests/test_exit.sh
new file mode 100755
index 0000000..2ad2699
--- /dev/null
+++ b/commands/tests/test_exit.sh
@@ -0,0 +1,3 @@
+#!./cloyster
+
+exit 42
diff --git a/lib/datetime.sh b/lib/datetime.sh
new file mode 100644
index 0000000..7f67a49
--- /dev/null
+++ b/lib/datetime.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# date_iso_utc
+date_now_iso_utc() {
+  TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ"
+}
+
+posix_time_microseconds() {
+  bc -e "($(perl -MTime::HiRes=gettimeofday -MPOSIX=strftime -e '($s,$us) = gettimeofday(); printf "%d.%06d\n", $s, $us') * 1000)" | cut -d '.' -f 1
+}
diff --git a/lib/dependencies.sh b/lib/dependencies.sh
new file mode 100644
index 0000000..bb86be3
--- /dev/null
+++ b/lib/dependencies.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+
+ensure_command_exists() {
+  _cmd=$1
+  _ret=0
+  if [ -z "${_cmd}" ]; then
+    _execution_error error=missing_argument function=ensure_command_exists argument=cmd
+  fi
+  if ! which "$1" > /dev/null 2>&1; then
+    _execution_error error=missing_dependency dependency="${_cmd}"
+  fi
+}
diff --git a/lib/identifiers.sh b/lib/identifiers.sh
new file mode 100644
index 0000000..cc7bc01
--- /dev/null
+++ b/lib/identifiers.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+
+# make_random_identifier [SIZE=32]
+# Generate a random alphanumeric identifier using `/dev/urandom`.
+# id=$(make_random_identifier)
+# id=$(make_random_identifier 64)
+make_random_identifier() {
+  _size="${1:-32}"
+  </dev/urandom env LC_CTYPE=C tr -dc "[:alnum:]" | head -c "$_size"
+}
diff --git a/lib/os.sh b/lib/os.sh
new file mode 100644
index 0000000..211bfaa
--- /dev/null
+++ b/lib/os.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env sh
+set -e
+
+export JSON_OS=
+
+if [ -f /etc/os-release ]; then
+  . /etc/os-release
+  _os_type="${NAME}"
+  _os_name="${ID}"
+  _os_pretty_name="${PRETTY_NAME}"
+  _os_version="${VERSION}"
+  _os_version_id="${VERSION_ID}"
+  _os_supported=1
+else
+  uname=$(uname | tr '[:upper:]' '[:lower:]')
+  case "${uname}" in
+    darwin)
+      _os_type="macos"
+      >&2 echo "${0}: error: macos not done yet!"
+      exit 1;;
+   *)
+      _os_type="${uname}"
+      >&2 echo "cloyster/os_detect.sh: error: unknown operating system: '${uname}'"
+  esac
+fi
+
+_m=$(uname -m)
+case "${_m}" in
+x86_64) _os_machine=amd64;;
+i*86) _os_machine=x86;;
+*)
+  _os_machine="${_m}"
+    ;;
+esac
+
+JSON_OS=$(jo -d. \
+  "os.supported@${_os_supported}" \
+   "os.type=${_os_type}" \
+   "os.name=${_os_name}" \
+   "os.pretty_name=${_os_pretty_name}" \
+   "os.version=${_os_version}" \
+   "os.version_id=${_os_version_id}" \
+   "os.machine=${_os_machine}")
+
+# os_get KEY|jq [JQ_QUERY]
+os_get() {
+  _key="${1}"
+  if [ -z "${1}" ]; then
+    _key=".os"
+  elif [ "${1}" = "jq" ] && [ -n "${2}" ]; then
+    _key="${2}"
+  else
+    _key=".os.${_key}"
+  fi
+  value="$(echo ${JSON_OS} | jq -r "${_key}")"
+  if [ "${value}" = "null" ]; then value=; fi
+  echo "${value}"
+}