From 93d3514676cad95b94bbb3e483d02b7ea0076bba Mon Sep 17 00:00:00 2001 From: Jordan Bracco Date: Sun, 17 Apr 2022 03:25:09 +0000 Subject: WIP --- apps/dreki/include/dreki_otel.hrl | 11 + apps/dreki/include/dreki_plum.hrl | 6 +- apps/dreki/src/dreki.app.src | 6 + apps/dreki/src/dreki_app.erl | 8 +- apps/dreki/src/dreki_config.erl | 26 +- apps/dreki/src/dreki_dets_store.erl | 74 ---- apps/dreki/src/dreki_freebsd_schemas.erl | 59 +++ apps/dreki/src/dreki_node.erl | 103 ----- apps/dreki/src/dreki_node_server.erl | 21 - apps/dreki/src/dreki_plum.erl | 30 +- apps/dreki/src/dreki_store.erl | 429 ------------------- apps/dreki/src/dreki_store_backend.erl | 33 -- apps/dreki/src/dreki_store_namespace.erl | 14 - apps/dreki/src/dreki_sup.erl | 10 +- apps/dreki/src/dreki_task.erl | 58 --- apps/dreki/src/dreki_tasks.erl | 141 ------ apps/dreki/src/dreki_tasks_cloyster.erl | 21 - apps/dreki/src/dreki_tasks_script.erl | 24 -- apps/dreki/src/dreki_world.erl | 366 ---------------- apps/dreki/src/dreki_world_dns.erl | 162 ------- apps/dreki/src/dreki_world_plum_events.erl | 17 - apps/dreki/src/dreki_world_server.erl | 63 --- apps/dreki/src/dreki_world_store.erl | 48 --- apps/dreki/src/dreki_world_tasks.erl | 66 --- apps/dreki/src/drekid.erl | 95 +++++ apps/dreki/src/drekid_tasks_store.erl | 71 +++ apps/dreki/src/funs/dreki_fun_exec.erl | 28 ++ apps/dreki/src/funs/dreki_funs.erl | 31 ++ apps/dreki/src/node/dreki_node.erl | 150 +++++++ apps/dreki/src/node/dreki_node_server.erl | 21 + apps/dreki/src/node/rebar.conf | 0 apps/dreki/src/requests/dreki_requests.erl | 97 +++++ apps/dreki/src/runs/dreki_runs.erl | 1 + apps/dreki/src/storages/dreki_storages.erl | 98 +++++ apps/dreki/src/storages/dreki_storages_zfs.erl | 2 + apps/dreki/src/store/dreki_dets_store.erl | 73 ++++ apps/dreki/src/store/dreki_mnesia_store.erl | 94 ++++ apps/dreki/src/store/dreki_store.erl | 475 +++++++++++++++++++++ apps/dreki/src/store/dreki_store_backend.erl | 33 ++ apps/dreki/src/store/dreki_store_namespace.erl | 14 + apps/dreki/src/tasks/dreki_task.erl | 58 +++ apps/dreki/src/tasks/dreki_tasks.erl | 166 +++++++ apps/dreki/src/tasks/dreki_tasks_cloyster.erl | 21 + apps/dreki/src/tasks/dreki_tasks_script.erl | 24 ++ apps/dreki/src/world/dreki_world.erl | 366 ++++++++++++++++ apps/dreki/src/world/dreki_world_dns.erl | 162 +++++++ apps/dreki/src/world/dreki_world_plum_events.erl | 17 + apps/dreki/src/world/dreki_world_server.erl | 63 +++ apps/dreki/src/world/dreki_world_store.erl | 48 +++ apps/dreki/src/world/dreki_world_tasks.erl | 66 +++ apps/dreki_web/.gitignore | 1 + apps/dreki_web/assets/package-lock.json | 13 +- apps/dreki_web/assets/package.json | 3 +- apps/dreki_web/rebar.config | 3 +- apps/dreki_web/src/admin/dreki_web_admin_tasks.erl | 30 ++ apps/dreki_web/src/admin/dreki_web_admin_world.erl | 28 ++ apps/dreki_web/src/api/dreki_web_index.erl | 9 + apps/dreki_web/src/api/dreki_web_task.erl | 21 + apps/dreki_web/src/cowboy_access_log_h.erl | 238 +++++++++++ apps/dreki_web/src/dreki_web.app.src | 3 +- apps/dreki_web/src/dreki_web_admin_tasks.erl | 30 -- apps/dreki_web/src/dreki_web_admin_world.erl | 28 -- apps/dreki_web/src/dreki_web_app.erl | 12 +- apps/dreki_web/src/dreki_web_handler.erl | 99 +++++ apps/dreki_web/src/dreki_web_index.erl | 9 - apps/dreki_web/src/dreki_web_task.erl | 21 - apps/dreki_web/src/dreki_web_ui.erl | 43 -- apps/dreki_web/src/dreki_web_ui_error.erl | 14 - apps/dreki_web/src/dreki_web_ui_index.erl | 31 -- apps/dreki_web/src/dreki_web_ui_json_form.erl | 127 ------ apps/dreki_web/src/dreki_web_ui_node.erl | 23 - apps/dreki_web/src/dreki_web_ui_stores.erl | 126 ------ apps/dreki_web/src/dreki_web_ui_task.erl | 20 - apps/dreki_web/src/dreki_web_ui_tasks.erl | 14 - apps/dreki_web/src/ui/dreki_web_ui.erl | 44 ++ apps/dreki_web/src/ui/dreki_web_ui_error.erl | 14 + apps/dreki_web/src/ui/dreki_web_ui_index.erl | 31 ++ apps/dreki_web/src/ui/dreki_web_ui_json_form.erl | 160 +++++++ apps/dreki_web/src/ui/dreki_web_ui_node.erl | 23 + apps/dreki_web/src/ui/dreki_web_ui_stores.erl | 189 ++++++++ apps/dreki_web/src/ui/dreki_web_ui_task.erl | 20 + apps/dreki_web/src/ui/dreki_web_ui_tasks.erl | 14 + apps/dreki_web/templates/crash.dtl | 52 +++ apps/dreki_web/templates/layout.dtl | 7 +- apps/dreki_web/templates/store_list.dtl | 8 +- apps/dreki_web/templates/store_new.dtl | 10 +- apps/dreki_web/templates/store_show.dtl | 8 + config/mgmt2/sys.config | 114 +++++ config/mgmt2/vm.args | 9 + config/sys.config | 56 +-- rebar.config | 11 +- rebar.lock | 45 +- 92 files changed, 3625 insertions(+), 2206 deletions(-) create mode 100644 apps/dreki/include/dreki_otel.hrl delete mode 100644 apps/dreki/src/dreki_dets_store.erl create mode 100644 apps/dreki/src/dreki_freebsd_schemas.erl delete mode 100644 apps/dreki/src/dreki_node.erl delete mode 100644 apps/dreki/src/dreki_node_server.erl delete mode 100644 apps/dreki/src/dreki_store.erl delete mode 100644 apps/dreki/src/dreki_store_backend.erl delete mode 100644 apps/dreki/src/dreki_store_namespace.erl delete mode 100644 apps/dreki/src/dreki_task.erl delete mode 100644 apps/dreki/src/dreki_tasks.erl delete mode 100644 apps/dreki/src/dreki_tasks_cloyster.erl delete mode 100644 apps/dreki/src/dreki_tasks_script.erl delete mode 100644 apps/dreki/src/dreki_world.erl delete mode 100644 apps/dreki/src/dreki_world_dns.erl delete mode 100644 apps/dreki/src/dreki_world_plum_events.erl delete mode 100644 apps/dreki/src/dreki_world_server.erl delete mode 100644 apps/dreki/src/dreki_world_store.erl delete mode 100644 apps/dreki/src/dreki_world_tasks.erl create mode 100644 apps/dreki/src/drekid.erl create mode 100644 apps/dreki/src/drekid_tasks_store.erl create mode 100644 apps/dreki/src/funs/dreki_fun_exec.erl create mode 100644 apps/dreki/src/funs/dreki_funs.erl create mode 100644 apps/dreki/src/node/dreki_node.erl create mode 100644 apps/dreki/src/node/dreki_node_server.erl create mode 100644 apps/dreki/src/node/rebar.conf create mode 100644 apps/dreki/src/requests/dreki_requests.erl create mode 100644 apps/dreki/src/runs/dreki_runs.erl create mode 100644 apps/dreki/src/storages/dreki_storages.erl create mode 100644 apps/dreki/src/storages/dreki_storages_zfs.erl create mode 100644 apps/dreki/src/store/dreki_dets_store.erl create mode 100644 apps/dreki/src/store/dreki_mnesia_store.erl create mode 100644 apps/dreki/src/store/dreki_store.erl create mode 100644 apps/dreki/src/store/dreki_store_backend.erl create mode 100644 apps/dreki/src/store/dreki_store_namespace.erl create mode 100644 apps/dreki/src/tasks/dreki_task.erl create mode 100644 apps/dreki/src/tasks/dreki_tasks.erl create mode 100644 apps/dreki/src/tasks/dreki_tasks_cloyster.erl create mode 100644 apps/dreki/src/tasks/dreki_tasks_script.erl create mode 100644 apps/dreki/src/world/dreki_world.erl create mode 100644 apps/dreki/src/world/dreki_world_dns.erl create mode 100644 apps/dreki/src/world/dreki_world_plum_events.erl create mode 100644 apps/dreki/src/world/dreki_world_server.erl create mode 100644 apps/dreki/src/world/dreki_world_store.erl create mode 100644 apps/dreki/src/world/dreki_world_tasks.erl create mode 100644 apps/dreki_web/src/admin/dreki_web_admin_tasks.erl create mode 100644 apps/dreki_web/src/admin/dreki_web_admin_world.erl create mode 100644 apps/dreki_web/src/api/dreki_web_index.erl create mode 100644 apps/dreki_web/src/api/dreki_web_task.erl create mode 100644 apps/dreki_web/src/cowboy_access_log_h.erl delete mode 100644 apps/dreki_web/src/dreki_web_admin_tasks.erl delete mode 100644 apps/dreki_web/src/dreki_web_admin_world.erl create mode 100644 apps/dreki_web/src/dreki_web_handler.erl delete mode 100644 apps/dreki_web/src/dreki_web_index.erl delete mode 100644 apps/dreki_web/src/dreki_web_task.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_error.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_index.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_json_form.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_node.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_stores.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_task.erl delete mode 100644 apps/dreki_web/src/dreki_web_ui_tasks.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_error.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_index.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_json_form.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_node.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_stores.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_task.erl create mode 100644 apps/dreki_web/src/ui/dreki_web_ui_tasks.erl create mode 100644 apps/dreki_web/templates/crash.dtl create mode 100644 config/mgmt2/sys.config create mode 100644 config/mgmt2/vm.args diff --git a/apps/dreki/include/dreki_otel.hrl b/apps/dreki/include/dreki_otel.hrl new file mode 100644 index 0000000..e784874 --- /dev/null +++ b/apps/dreki/include/dreki_otel.hrl @@ -0,0 +1,11 @@ +-include_lib("opentelemetry_api/include/opentelemetry.hrl"). +-include_lib("opentelemetry_api/include/otel_tracer.hrl"). + +-define(FUN_NAME, binary:list_to_bin([ + atom_to_binary(?MODULE), <<":">>, + atom_to_binary(?FUNCTION_NAME), <<"/">>, + integer_to_binary(?FUNCTION_ARITY) + ]) +). + +-define(FUN_NAME(D), binary:list_to_bin([?FUN_NAME, atom_to_binary(D)])). diff --git a/apps/dreki/include/dreki_plum.hrl b/apps/dreki/include/dreki_plum.hrl index bea65aa..aed2e05 100644 --- a/apps/dreki/include/dreki_plum.hrl +++ b/apps/dreki/include/dreki_plum.hrl @@ -5,14 +5,13 @@ %% dreki_store tabs -define(PLUM_DB_STORE_TASKS_TAB, dreki_tasks). - %% indices tabs -define(PLUM_DB_IDX_ROLE_TAB, 'dreki_idx:roles'). -define(PLUM_DB_IDX_TAGS_TAB, 'dreki_idx:tags'). -define(PLUM_DB_PREFIXES, [ - {?PLUM_DB_REGIONS_TAB, ram_disk}, - {?PLUM_DB_NODES_TAB, ram_disk}, + {?PLUM_DB_REGIONS_TAB, disk}, + {?PLUM_DB_NODES_TAB, disk}, {?PLUM_DB_PATHS_TAB, ram_disk}, {?PLUM_DB_STORES_TAB, ram_disk}, @@ -23,4 +22,3 @@ {?PLUM_DB_IDX_ROLE_TAB, disk}, {?PLUM_DB_IDX_TAGS_TAB, disk} ]). - diff --git a/apps/dreki/src/dreki.app.src b/apps/dreki/src/dreki.app.src index 78009bf..2e152a7 100644 --- a/apps/dreki/src/dreki.app.src +++ b/apps/dreki/src/dreki.app.src @@ -8,9 +8,15 @@ stdlib, mnesia, logger_colorful, + ipee, opentelemetry, opentelemetry_api, opentelemetry_exporter, + opentelemetry_logger_metadata, + telemetry, + opentelemetry_telemetry, + prometheus, + genlib, uuid, mnesia_rocksdb, plum_db, diff --git a/apps/dreki/src/dreki_app.erl b/apps/dreki/src/dreki_app.erl index a1259a7..0442b4c 100644 --- a/apps/dreki/src/dreki_app.erl +++ b/apps/dreki/src/dreki_app.erl @@ -31,6 +31,8 @@ before_start(Type, Args) -> logger:set_application_level(dreki_web, debug), logger:set_application_level(partisan, info), logger:set_application_level(plum_db, info), + logger:set_handler_config(default, level, debug), + opentelemetry_logger_metadata:setup(), ?LOG_NOTICE(#{message => "Dreki starting...."}), application:stop(partisan), ok = dreki_config:init(Args), @@ -48,6 +50,7 @@ after_start() -> ok = setup_event_manager(), ok = dreki_store:start(), ?LOG_NOTICE(#{message => "Dreki Ready"}), + ok = dreki_config:set([dreki, status], ready), dreki_event_manager:notify(dreki_ready), ok. @@ -70,12 +73,11 @@ setup_event_manager() -> Mod = partisan_peer_service:manager(), Mod:on_up('_', fun(Node) -> - dreki_event_manager:notify({peer_up, Node}) + dreki_event_manager:notify({partisan_peer_service, peer_up, Node}) end), Mod:on_down('_', fun(Node) -> - dreki_event_manager:notify({peer_down, Node}) + dreki_event_manager:notify({partisan_peer_service, peer_down, Node}) end), ok. - diff --git a/apps/dreki/src/dreki_config.erl b/apps/dreki/src/dreki_config.erl index 69df2d6..457f559 100644 --- a/apps/dreki/src/dreki_config.erl +++ b/apps/dreki/src/dreki_config.erl @@ -2,8 +2,14 @@ -include_lib("kernel/include/logger.hrl"). -include_lib("partisan/include/partisan.hrl"). -include("dreki_plum.hrl"). +-include("dreki_otel.hrl"). + +-compile({no_auto_import,[get/0]}). + -export([init/1]). +-export([get/1]). +-export([set/2]). -define(CONFIG, [ %% All stolen from bondy as partisan isn't that well documented, eh. @@ -44,15 +50,29 @@ -define(PT, dreki_config_cache). init(_Args) -> - persistent_term:put(?PT, application:get_all_env(dreki)), + persistent_term:put(?PT, [{dreki, application:get_all_env(dreki)}, + {dreki_web, application:get_all_env(dreki_web) + }]), ok = set_app_configs(?CONFIG), - ?LOG_INFO(#{message => "Configured Dreki and dependencies"}), + ?LOG_NOTICE(#{message => "Configured Dreki and dependencies"}), ok = partisan_config:init(), ok. +get() -> + persistent_term:get(?PT). + +get(Key) -> + ?with_span(?FUN_NAME, #{}, fun (_) -> key_value:get(Key, get()) end). + +set(Key, Value) -> + ?with_span(?FUN_NAME, #{}, + fun (_) -> + persistent_term:put(?PT, key_value:put(Key, Value, get())), + ok + end). + set_app_configs(Configs) -> lists:foreach(fun ({App, Params}) -> [application:set_env(App, Key, Value) || {Key, Value} <- Params] end, Configs), ok. - diff --git a/apps/dreki/src/dreki_dets_store.erl b/apps/dreki/src/dreki_dets_store.erl deleted file mode 100644 index 05cf9fb..0000000 --- a/apps/dreki/src/dreki_dets_store.erl +++ /dev/null @@ -1,74 +0,0 @@ --module(dreki_dets_store). - --include("dreki.hrl"). - --behaviour(dreki_store_backend). - --export([start/0, start/4, checkout/1, checkin/1, stop/0, stop/1]). --export([valid_store/5]). --export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). - --record(?MODULE, {tab, file}). - -start() -> ok. - -valid_store(_Namespace, _Location, _Name, _NSMod, _Args) -> ok. - -start(Namespace, Name, _XUrn, Args) -> - StoreName = {?MODULE, Namespace, Name}, - File = maps:get(file_name, Args, <>), - FileName = binary:bin_to_list(File), - case dets:open_file(StoreName, [{file, FileName}, {keypos, 2}]) of - {ok, Tab} -> {ok, #?MODULE{tab = Tab, file = FileName}}; - {error, Error} -> {error, {dets_open_failed, Error}} - end. - -checkout(#?MODULE{tab = Tab, file = FileName}) -> - case dets:open_file(Tab, [{file, FileName}, {keypos, 2}]) of - {ok, Tab} -> {ok, {dreki_dets_store_ref, Tab}}; - {error, Error} -> {error, {dets_open_failed, Error}} - end. - -checkin({dreki_dets_store_ref, Tab}) -> - dets:close(Tab). - -stop() -> ok. - -stop(#?MODULE{tab = Tab}) -> - dets:close(Tab). - -list({dreki_dets_store_ref, Tab}) -> - dets:foldl(fun (T, {ok, Ts}) -> {ok, [T | Ts]} end, {ok, []}, Tab). - -count({dreki_dets_store_ref, Tab}) -> - dets:foldl(fun (_T, {ok, Ct}) -> {ok, Ct + 1} end, {ok, 0}, Tab). - -exists({dreki_dets_store_ref, Tab}, Id) -> - dets:member(Tab, Id). - -get({dreki_dets_store_ref, Tab}, Id) -> - case dets:lookup(Tab, Id) of - [] -> {error, {task_not_found, Id}}; - [Task] -> {ok, Task} - end. - -delete({dreki_dets_store_ref, Tab}, Id) -> - dets:delete(Tab, Id). - -create({dreki_dets_store_ref, Tab}, Task = #dreki_task{persisted=false}) -> - case dets:insert_new(Tab, Task#dreki_task{persisted=true, dirty=false}) of - true -> {ok, Task#dreki_task{persisted=true, dirty=false}}; - false -> {error, {task_exists, Task#dreki_task.id}}; - {error, Error} -> {error, Error} - end. - -update(_Tab, Task = #dreki_task{persisted=false}) -> - {error, {task_not_created, Task#dreki_task.id}}; -update(_Tab, Task = #dreki_task{dirty=false}) -> - {ok, Task}; -update({dreki_dets_store_ref, Tab}, Task = #dreki_task{persisted=true, dirty=true}) -> - case dets:insert(Tab, Task#dreki_task{dirty=false}) of - ok -> {ok, Task#dreki_task{dirty=false}}; - {error, Error} -> {error, Error} - end. - diff --git a/apps/dreki/src/dreki_freebsd_schemas.erl b/apps/dreki/src/dreki_freebsd_schemas.erl new file mode 100644 index 0000000..de97ded --- /dev/null +++ b/apps/dreki/src/dreki_freebsd_schemas.erl @@ -0,0 +1,59 @@ +-module(dreki_freebsd_schemas). + +-export([schema/1]). + +schema(<<"dreki.v1.freebsd.jails.list">>) -> + #{ + <<"@schema">> => "dreki.v1.freebsd.jails.list", + version => 'draft-06', + title => <<"List running jails ID/Names">>, + properties => #{} + }; +schema(<<"dreki.v1.freebsd.jails.get">>) -> + #{ + <<"@schema">> => "dreki.v1.freebsd.jails.get", + version => 'draft-06', + title => <<"Get jail information">>, + properties => #{ + jail_id => #{type => integer} + }, + required => [jail_id] + }; +schema(<<"dreki.v1.freebsd.jails.start">>) -> + #{ + <<"@schema">> => "dreki.v1.freebsd.jails.start", + version => 'draft-06', + title => <<"Start a jail">>, + properties => #{ + name => #{type => string}, + path => #{type => string}, + hostname => #{type => string}, + params => #{type => object} + }, + required => [name, path, hostname, params] + }; +schema(<<"dreki.v1.freebsd.jails.exec">>) -> + #{ + <<"@schema">> => "dreki.v1.freebsd.jails.exec", + version => 'draft-06', + title => <<"Execute a command in a jail">>, + properties => #{ + jail_id => #{type => string}, + command => #{type => string}, + args => #{type => array}, + env => #{type => object} + }, + required => [id, command, args, env] + }; +schema(<<"dreki.v1.exec">>) -> + #{ + <<"@schema">> => "dreki.v1.exec", + version => 'draft-06', + title => <<"Execute a command">>, + properties => #{ + command => #{type => string}, + args => #{type => array}, + env => #{type => object} + }, + required => [command, args, env] + }. diff --git a/apps/dreki/src/dreki_node.erl b/apps/dreki/src/dreki_node.erl deleted file mode 100644 index 87dbc73..0000000 --- a/apps/dreki/src/dreki_node.erl +++ /dev/null @@ -1,103 +0,0 @@ --module(dreki_node). --include("dreki.hrl"). --include_lib("opentelemetry_api/include/otel_tracer.hrl"). --behaviour(partisan_gen_fsm). --compile({no_auto_import,[get/0]}). --export([get/0, urn/0, stores/0]). --export([rpc/4, rpc/5]). --export([uri/0]). % deprecated --export([parents/0, parents/1, parent/0, parent/1]). --export([neighbours/0, neighbours/1]). --export([descendants/0, descendants/1]). --export([ensure_local_node/0]). - --type rpc_error() :: {rpc_error, dreki_urn(), timeout | any()}. - -rpc(Path, Mod, Fun, Args) -> - rpc(Path, Mod, Fun, Args, #{timeout => 1000}). - --spec rpc(dreki_urn(), module(), function(), Args :: [], #{timeout => non_neg_integer()}) -> {ok, any()} | {error, rpc_error()}. -rpc(Path, Mod, Fun, Args, #{timeout := Timeout}) -> - ?with_span(<<"dreki_node:rpc">>, #{}, fun(ChildSpanCtx) -> - case dreki_world_dns:node_param(dreki_world:path_to_domain(Path), node_name) of - {ok, NodeName} -> - case partisan_rpc_backend:call(NodeName, Mod, Fun, Args, Timeout) of - {badrpc, Error} -> {error, {rpc_error, Path, Error}}; - Result -> Result - end; - Error -> Error - end - end). - -get() -> - {ok, Node} = dreki_world:get_node(dreki_world:node()), - Node. - -urn() -> dreki_world:path(). - -uri() -> urn(). - -stores() -> dreki_world:stores(uri()). - -parents() -> - parents(urn()). - -parents(Node) -> - {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), - NodeDomain = dreki_world:path_to_domain(NodeUrn), - Vertices0 = dreki_world_dns:get_path({root, dreki_world:internal_domain()}, {node, NodeDomain}), - Vertices = lists:map(fun get_from_vertex/1, Vertices0), - [_Me | Parents] = lists:reverse(Vertices), - Parents. - -parent() -> - parent(urn()). - -parent(Node) -> - {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), - NodeDomain = dreki_world:path_to_domain(NodeUrn), - [Parent] = dreki_world_dns:in_neighbours({node, NodeDomain}), - get_from_vertex(Parent). - -neighbours() -> - neighbours(urn()). - -neighbours(Node) -> - {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), - NodeDomain = dreki_world:path_to_domain(NodeUrn), - [Parent] = dreki_world_dns:in_neighbours({node, NodeDomain}), - Neighbours = dreki_world_dns:out_neighbours(Parent), - lists:map(fun get_from_vertex/1, Neighbours -- [{node, NodeDomain}]). - -descendants() -> - descendants(urn()). - -descendants(Node) -> - {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), - NodeDomain = dreki_world:path_to_domain(NodeUrn), - Descendants0 = dreki_world_dns:out_neighbours({node, NodeDomain}), - Descendants = case Descendants0 of - [{region, NodeDomain}] -> dreki_world_dns:out_neighbours({region, NodeDomain}); - D -> D - end, - lists:map(fun get_from_vertex/1, Descendants). - -get_from_vertex({root, Domain}) -> - {ok, Region} = dreki_world:get_region_from_dns_name(Domain), - Region; -get_from_vertex({region, Domain}) -> - {ok, Region} = dreki_world:get_region_from_dns_name(Domain), - Region; -get_from_vertex({node, Domain}) -> - {ok, Node} = dreki_world:get_node_from_dns_name(Domain), - Node. - -ensure_local_node() -> - case dreki_world:get_node(uri()) of - {ok, _} -> ok; - {error, {not_found, _}} -> create_local_node() - end. - -create_local_node() -> - dreki_world:create_node(uri(), #{}). - diff --git a/apps/dreki/src/dreki_node_server.erl b/apps/dreki/src/dreki_node_server.erl deleted file mode 100644 index c8bca51..0000000 --- a/apps/dreki/src/dreki_node_server.erl +++ /dev/null @@ -1,21 +0,0 @@ --module(dreki_node_server). --behaviour(partisan_gen_fsm). - --export([start_link/1, send_event/2]). --export([init/1]). --export([wait/2]). - --record(data, { }). - -start_link(Args) -> - partisan_gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). - -send_event(Name, Event) -> - partisan_gen_fsm:send_event(Name, Event). - -init(_) -> - {ok, wait, #data{}}. - -wait(Event, Data) -> - logger:info("node_server wait event: ~p", [Event]), - {next_state, wait, Data}. diff --git a/apps/dreki/src/dreki_plum.erl b/apps/dreki/src/dreki_plum.erl index 9c03e46..dccdad7 100644 --- a/apps/dreki/src/dreki_plum.erl +++ b/apps/dreki/src/dreki_plum.erl @@ -8,7 +8,6 @@ before_start() -> %% We temporarily disable plum_db's AAE to avoid rebuilding hashtrees %% until we are ready to do it ok = suspend_aae(), - logger:debug("-- DISABLED AAE ! --"), _ = application:ensure_all_started(plum_db, permanent), ok. @@ -27,9 +26,7 @@ suspend_aae() -> true -> ok = application:set_env(plum_db, priv_aae_enabled, true), ok = application:set_env(plum_db, aae_enabled, false), - ?LOG_NOTICE(#{ - description => "Temporarily disabled active anti-entropy (AAE) during initialisation" - }), + ?LOG_NOTICE(#{message => "Temporarily disabled plum_db aae during initialisation"}), ok; false -> ok @@ -40,9 +37,7 @@ restore_aae() -> true -> %% plum_db should have started so we call plum_db_config ok = plum_db_config:set(aae_enabled, true), - ?LOG_NOTICE(#{ - description => "Active anti-entropy (AAE) re-enabled" - }), + ?LOG_NOTICE(#{message => "plum_db aae re-enabled"}), ok; false -> ok @@ -51,11 +46,10 @@ restore_aae() -> maybe_wait_for_plum_db_partitions() -> case wait_for_partitions() of true -> - %% We block until all partitions are initialised - ?LOG_NOTICE(#{ - description => "Application master is waiting for plum_db partitions to be initialised" - }), - plum_db_startup_coordinator:wait_for_partitions(); + ?LOG_NOTICE(#{domain => [dreki_plum], message => "Waiting for plum_db partitions to be initialised"}), + ok = plum_db_startup_coordinator:wait_for_partitions(), + ?LOG_NOTICE(#{domain => [dreki_plum], message => "plum_db partitions initialised"}), + ok; false -> ok end. @@ -64,10 +58,10 @@ maybe_wait_for_plum_db_hashtrees() -> case wait_for_hashtrees() of true -> %% We block until all hashtrees are built - ?LOG_NOTICE(#{ - description => "Application master is waiting for plum_db hashtrees to be built" - }), - plum_db_startup_coordinator:wait_for_hashtrees(); + ?LOG_NOTICE(#{domain => [dreki_plum], message => "Waiting for plum_db hashtrees to be built"}), + ok = plum_db_startup_coordinator:wait_for_hashtrees(), + ?LOG_NOTICE(#{domain => [dreki_plum], message => "plum_db hashtrees built"}), + ok; false -> ok end, @@ -88,9 +82,7 @@ maybe_wait_for_aae_exchange() -> %% We have not yet joined a cluster, so we finish ok; Peers -> - ?LOG_NOTICE(#{ - description => "Application master is waiting for plum_db AAE to perform exchange" - }), + ?LOG_NOTICE(#{domain => [plum_db], message => "Waiting for plum_db AAE to perform exchange"}), %% We are in a cluster, we randomnly pick a peer and %% perform an AAE exchange [Peer|_] = lists_utils:shuffle(Peers), diff --git a/apps/dreki/src/dreki_store.erl b/apps/dreki/src/dreki_store.erl deleted file mode 100644 index ae6fd66..0000000 --- a/apps/dreki/src/dreki_store.erl +++ /dev/null @@ -1,429 +0,0 @@ --module(dreki_store). --include("dreki.hrl"). --include("dreki_plum.hrl"). --include_lib("opentelemetry_api/include/otel_tracer.hrl"). --define(BACKENDS_PT, {dreki_stores, backends}). --compile({no_auto_import,[get/1]}). --export([backends/0]). --export([start/0]). --export([namespaces/0, namespace/1]). --export([stores/0, stores_local/0, stores/1, get_store/1, create_store/4, create_store/6]). --export([list/1, get/1, new/1, create/2, update/2, delete/1]). --export([store_as_map/1]). - --behaviour(dreki_urn). --export([expand_urn_resource_rest/4]). - --export([callback/3, list_/2, get_/2, new_/2, create_/3, update_/3, delete_/2]). - --record(store, {urn, xurn, namespace, name, namespace_mod, backend_mod, backend_params}). - --type t() :: #store{}. --type store() :: t() | dreki_uri() | dreki_expanded_uri(). - -backends() -> [dreki_dets_store, dreki_world_store]. -namespaces() -> [{<<"tasks">>, dreki_tasks, #{}}]. - -namespace(Name) -> - lists:keyfind(Name, 1, namespaces()). - -start() -> - [ok = dreki_urn:register_namespace(NS, ?MODULE, Env) || {NS, _, Env} <- namespaces()], - [ok = BackendMod:start() || BackendMod <- backends()], - persistent_term:put(?BACKENDS_PT, #{}), - {Backends, Errors} = lists:foldr(fun (Store = #store{}, {Acc, Errs}) -> - case start_store_(Store) of - {ok, Backend} -> {maps:put(Store#store.urn, Backend, Acc), Errs}; - {error, Err} -> {Acc, [Err | Errs]} - end - end, {#{}, []}, stores_local()), - persistent_term:put(?BACKENDS_PT, Backends), - [logger:error("dreki_store: ~p", [Err]) || Err <- Errors], - ok. - -stores() -> - plum_db:fold(fun - ({_, Value}, Acc) -> [as_record(Value) | Acc]; - ({_, [Value]}, Acc) -> [as_record(Value) | Acc]; - ({_, ['$deleted']}, Acc) -> Acc - end, [], {?PLUM_DB_STORES_TAB, '_'}). - -stores_local() -> - lists:filter(fun is_local/1, stores()). - -stores(Namespace) -> - case namespace(Namespace) of - undefined -> {error, {namespace_not_found, Namespace}}; - {_, _, _} -> {ok, [Store || Store = #store{namespace = Namespace} <- stores()]} - end. - -start_store(Store = #store{}) -> - case start_store_(Store) of - {ok, Backend} -> - alarm_handler:clear_alarm({?MODULE, Store#store.urn}), - Pt = persistent_term:get(?BACKENDS_PT), - persistent_term:put(?BACKENDS_PT, maps:put(Store#store.urn, Backend, Pt)), - {ok, Backend}; - Error -> - alarm_handler:set_alarm({?MODULE, Store#store.urn}, {start_failed, Error}), - Error - end. - -start_store_(Store = #store{backend_mod = Mod}) -> - case is_local(Store) of - true -> start_store__(Store); - false -> {error, {store_not_local, Store#store.urn, dreki_node:urn()}} - end. -start_store__(Store) -> - case maps:get(Store#store.urn, persistent_term:get(?BACKENDS_PT), undefined) of - undefined -> start_store___(Store); - _Started -> {error, {store_already_started, Store#store.urn}} - end. -start_store___(Store = #store{backend_mod = Mod}) -> - case Mod:start(Store#store.namespace, Store#store.name, Store#store.urn, Store#store.backend_params) of - {ok, Backend} -> {ok, Backend}; - {error, Error} -> {error, {store_start_failed, Store#store.urn, Error}} - end. - -create_store(Urn, Module, ModuleParams, Params) when is_binary(Urn) -> - case dreki_urn:expand(Urn) of - Error = {error, _} -> Error; - {ok, XUrn} -> create_store(XUrn, Module, ModuleParams, Params) - end; -create_store(#{location := Location, resource := #{directory := #{namespace := NS, directory := Name}}}, Module, ModuleParams, Params) -> - create_store(NS, Location, Name, Module, ModuleParams, Params). - -create_store(Namespace, Location, Name, Module, ModuleParams, Params) -> - case lists:keyfind(Namespace, 1, namespaces()) of - undefined -> {error, {namespace_not_found, Namespace}}; - {_, NSMod, NSEnv} -> - case dreki_urn:expand(Location) of - Error = {error, _} -> Error; - {ok, #{location := Loc}} -> - Urn = <>, - case get_store(Urn) of - {ok, _} -> dreki_error:error(exists, [{urn, Urn}]); - Error = {error, _} -> Error; - not_found -> - case {NSMod:valid_store(Namespace, Location, Name, Module), Module:valid_store(Namespace, Location, Name, NSMod, ModuleParams)} of - {ok, ok} -> - {Prefix, Key} = {{?PLUM_DB_STORES_TAB, Loc}, {Namespace, Name}}, - %%ok = put_path(Urn, store, {Prefix, Key}), - ok = plum_db:put(Prefix, Key, #{urn => Urn, name => Name, namespace => Namespace, module => Module, module_params => ModuleParams, params => Params}), - get_store(Urn); - {{error, Error}, _} -> {error, {store_create_failed_namespace_rejected, {NSMod, Error}}}; - {_, {error, Error}} -> {error, {store_create_failed_store_rejected, {Module, Error}}} - end - end - end - end. - -expand_urn_resource_rest(Namespace, ResXUrn, Part, _Env) -> - logger:debug("Expanding resource ~p ~p", [Part, ResXUrn]), - expand_urn_resource_rest_(Namespace, ResXUrn, binary:split(Part, <<":">>, [global])). - -expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>]) -> - {ok, Res#{schemas => #{schemas => all}}}; -expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, Schema]) -> - {ok, Res#{schemas => #{schemas => Schema}}}; -expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, <<>>, <<>>]) -> - {ok, Res#{schema => #{schema => default}}}; -expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, Schema, SchemaVer]) -> - {ok, Res#{schema => #{schema => Schema, version => SchemaVer}}}; -expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>]) -> - {ok, Res#{schemas => #{schemas => all}}}; -expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, Schema]) -> - {ok, Res#{schemas => #{schemas => Schema}}}; -expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, <<>>, <<>>]) -> - {ok, Res#{schema => #{schema => default}}}; -expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, Schema, SchemaVer]) -> - {ok, Res#{schema => #{schema => Schema, version => SchemaVer}}}; -expand_urn_resource_rest_(_, _, _) -> - error. - --spec get_store(store()) -> {ok, t()} | {error, any()}. -get_store(Urn) when is_binary(Urn) -> - case dreki_urn:expand(Urn) of - {ok, Uri} -> get_store(Uri); - Error -> Error - end; -get_store(#{location := Location, resource := #{resource := #{namespace := NS, directory := Dir}}}) -> - get_store(Location, NS, Dir); -get_store(#{location := Location, resource := #{directory := #{namespace := NS, directory := Dir}}}) -> - get_store(Location, NS, Dir); -get_store(Fail) -> - {error, {uri_resolution_failed, Fail}}. - -get_store(Location, Namespace, Name) -> - case plum_db:get({?PLUM_DB_STORES_TAB, Location}, {Namespace, Name}) of - undefined -> not_found; - ['$deleted'] -> not_found; - [Val] -> {ok, as_record(Val)}; - Val -> {ok, as_record(Val)} - end. - -as_record([Map]) -> as_record(Map); -as_record(#{urn := Urn, name := Name, namespace := Namespace, module := Module, module_params := ModuleParams}) -> - {ok, XUrn} = dreki_urn:expand(Urn), - {_, NSMod, _} = lists:keyfind(Namespace, 1, namespaces()), - #store{urn = Urn, xurn = XUrn, name = Name, namespace = Namespace, namespace_mod = NSMod, backend_mod = Module, backend_params = ModuleParams}. - -store_as_map(#store{urn = Urn, name = Name, namespace = Namespace, namespace_mod = NSMod, backend_mod = Module, backend_params = ModuleParams}) -> - #{urn => Urn, name => Name, namespace => Namespace, namespace_mod => NSMod, backend_mod => Module, backend_params => ModuleParams}. - --spec list(store()) -> {ok, dreki_store_namespace:collection()} | {error, any()}. -list(SArg) -> - get_store_(SArg, list_, []). - -new(SArg) -> - get_store_(SArg, new_, []). - -get(SArg) -> - get_store_(SArg, get_, []). - -create(SArg, Data) -> - get_store_(SArg, create_, [Data]). - -update(SArg, Data) -> - get_store_(SArg, update_, [Data]). - -delete(SArg) -> - get_store_(SArg, delete_, []). - -list_(#{resource := #{directory := _}}, Store) -> - handle_result(Store, collection, callback(Store, list, [])); -list_(#{schema := _}, Store) -> - {todo, schema}; -list_(_, _) -> - not_supported. - -get_(#{resource := #{resource := #{id := Id}}}, Store) -> - handle_result(Store, single, callback(Store, get, [Id])); -get_(#{schema := _}, Store) -> - {todo, get_schema}; -get_(_, _) -> not_supported. - -new_(#{resource := #{directory := _}}, Store) -> - NSMod = Store#store.namespace_mod, - New = NSMod:new(), - {ok, New}. - -create_(XUrn = #{directory := _}, Store, Data) -> - case validate(XUrn, Data) of - {ok, _} -> - handle_result(Store, single, callback(Store, create, [Data])); - {error, Errors} -> {error, #{code => validation_failed, status => 422, errors => jesse_error:to_json({error, Errors}, [])}} - end; -create_(_, _, _) -> - not_supported. - -update_(XUrn = #{resource := _}, Store, Data) -> - case get(XUrn) of - {ok, Prev} -> - case validate(XUrn, Data) of - {ok, _} -> - handle_result(Store, single, callback(Store, create, [Data])); - {error, Errors} -> - {error, #{code => validation_failed, status => 422, errors => jesse_error:to_json({error, Errors}, [])}} - end; - Error -> Error - end; -update_(_, _, _) -> - not_suppported. - -delete_(#{resource := _}, Store) -> - todo; -delete_(_, _) -> - not_supported. - -get_store_(StoreArg, Fun, Args) when is_binary(StoreArg) -> - logger:debug("Expanding in get_store_ :) ~p", [StoreArg]), - case dreki_urn:expand(StoreArg) of - Error = {error, _} -> Error; - {ok, XUrn = #{resource := #{schemas := _}}} -> get_schema_(XUrn, Fun, Args); - {ok, XUrn = #{resource := #{schema := _}}} -> get_schema_(XUrn, Fun, Args); - {ok, XUrn} -> get_store_(XUrn, Fun, Args) - end; -get_store_(XUrn = #{resource := _}, Fun, Args) -> - ?with_span(<<"dreki_store:get_store_">>, #{}, fun(_Ctx) -> - logger:debug("Getting store usually! ~p", [XUrn]), - case get_store(XUrn) of - Error = {error, _} -> Error; - {ok, Store} -> - case apply(?MODULE, Fun, [XUrn, Store | Args]) of - not_supported -> {error, {not_supported, Fun, maps:get(urn, XUrn)}}; - Out -> Out - end - end - end). - -is_local(#store{xurn = #{kind := region}}) -> true; -is_local(#store{xurn = #{kind := node, location := Location}}) -> Location =:= dreki_node:uri(). - -% TODO: Rescue execution so we always checkin -callback(Store = #store{}, Fun, Args) -> - callback(is_local(Store), Store, Fun, Args). - -% TODO: Rescue execution so we always checkin -callback(false, Store = #store{xurn = #{location := Location}}, Fun, Args) -> - logger:debug("dreki_store:callback: rpc:~p ~p ~p", [Location, Fun, Args]), - dreki_node:rpc(Location, ?MODULE, callback, [Store, Fun, Args]); -callback(true, #store{urn = Urn, backend_mod = Mod, backend_params = Params}, Fun, Args) -> - logger:debug("dreki_store:callback: local ~p ~p", [Fun, Args]), - case maps:get(Urn, persistent_term:get(?BACKENDS_PT), undefined) of - undefined -> dreki_error:error(store_backend_not_started, 503, <<"Store backend is not started">>); - Backend -> - {ok, B} = Mod:checkout(Backend), - Res = apply(Mod, Fun, [B | Args]), - ok = Mod:checkin(B), - Res - end. - -handle_result(_, _, {error, Err}) -> {error, Err}; -handle_result(Store, collection, {ok, Collection}) when is_list(Collection) -> - handle_result(Store, collection, Collection); -handle_result(Store, collection, Collection) when is_list(Collection) -> - {ok, handle_collection_result(Collection, Store, [])}; -handle_result(Store, single, {ok, Item}) when is_map(Item) -> - {ok, handle_single_result(Item, Store)}. - -handle_collection_result([Item | Rest], Store, Acc0) -> - Acc = case handle_single_result(Item, Store) of - ignore -> Acc0; - {ok, I} -> [I | Acc0] - end, - handle_collection_result(Rest, Store, Acc); -handle_collection_result([], Store, Acc) -> - format_collection(Acc, Store). - -handle_single_result(Item = #{id := LId}, Store = #store{namespace_mod = Mod}) -> - case Mod:format_item(Item) of - ok -> format_item(Item, Store); - {ok, M} -> format_item(M, Store); - Err -> Err - end. - -format_item(Item = #{id := LId}, Store = #store{urn = Urn}) -> - AtLinks = #{ - self => <>, - parent => maps:get(location, Urn) - }, - AllAtLinks = maps:merge(AtLinks, maps:get('@links', Item, #{})), - I = #{'@id' => <>, '@links' => AllAtLinks}, - {ok, maps:merge(I, Item)}. - -format_collection(Data, Store = #store{urn = Urn}) -> - #{'@links' => #{self => Urn}, - data => Data}. - -get_schema_(XUrn = #{resource := #{directory := #{directory := _, namespace := NS}, schemas := Schemas}}, Fun, Args) -> - get_schema_(XUrn#{resource => #{namespace => NS, schemas => Schemas}}, Fun, Args); -get_schema_(XUrn = #{resource := #{directory := #{directory := _, namespace := NS}, schema := Schema}}, Fun, Args) -> - get_schema_(XUrn#{resource => #{namespace => NS, schema => Schema}}, Fun, Args); -get_schema_(#{urn := Urn, resource := #{namespace := NS, schemas := #{schemas := all}}}, list_, _) -> - {_, NSMod, _} = namespace(NS), - Schemas = maps:fold(fun - (default, Value, Acc) -> maps:put(default, Value, Acc); - (SchemaName, Content, Acc) -> - SUrn = <>, -logger:debug("Content is ~p", [Content]), - Schema = maps:fold(fun - (default_version, Value, CAcc) -> maps:put(default_version, Value, CAcc); - (Vsn, _, CAcc) when is_binary(Vsn) -> - Version = #{id => <>, version => Vsn}, - maps:put(versions, [Version | maps:get(versions, CAcc, [])], CAcc) - end, #{id => SUrn}, Content), - maps:put(schemas, [Schema | maps:get(schemas, Acc, [])], Acc) - end, #{}, NSMod:schemas()), - {ok, Schemas}; -get_schema_(#{urn := Urn, resource := #{namespace := NS, schemas := #{schemas := Query}}}, list_, _) -> - logger:debug("Querying Schemas ~p", [Query]), - {_, NSMod, _} = namespace(NS), - case maps:get(Query, NSMod:schemas(), not_found) of - not_found -> {error, not_found}; - SchemaSpec -> - Schema = maps:fold(fun - (default, Value, CAcc) -> maps:put(default_version, Value, CAcc); - (Vsn, _, CAcc) -> - Version = #{id => <>, version => Vsn}, - maps:put(versions, [Version | maps:get(versions, CAcc, [])], CAcc) - end, #{id => Urn}, SchemaSpec), - {ok, Schema} - end; -get_schema_(#{urn := Urn, resource := #{namespace := NS, schema := #{schema := default}}}, get_, Args) -> - logger:debug("Looking for default schema!"), - {_, NSMod, _} = namespace(NS), - Schemas = NSMod:schemas(), - SchemaName = maps:get(default, Schemas, not_found), - logger:debug("=+> SchemaName ~p IN ~p", [SchemaName, Schemas]), - case maps:get(SchemaName, Schemas, not_found) of - not_found -> not_found; - Schema -> - case maps:get(default_version, Schema, not_found) of - not_found -> not_found; - SchemaVer -> - NewUrn = binary:replace(Urn, <<"schemas::">>, <<"schemas:", SchemaName/binary, ":", SchemaVer/binary>>), - {ok, XUrn} = dreki_urn:expand(NewUrn), - logger:debug("Looking up default schema as ~p", [NewUrn, XUrn]), - get_schema_(XUrn, get_, Args) - end - end; -get_schema_(XUrn = #{resource := #{namespace := NS, schema := #{schema := SchemaName, version := Version}}}, get_, _) -> - {_, NSMod, _} = namespace(NS), - case maps:get(SchemaName, NSMod:schemas(), not_found) of - not_found -> not_found; - Schema -> format_schema(maps:get(Version, Schema, not_found), SchemaName, Version, XUrn) - end; -get_schema_(XUrn = #{urn := Urn}, Method, _) -> - {error, {unsupported, Urn, XUrn, Method}}. - -format_schema(not_found, _, _, _) -> - {error, not_found}; -format_schema(Schema, SchemaName, SchemaVersion, XUrn = #{urn := Urn}) -> - {ok, maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, SchemaName, SchemaVersion, XUrn), maps:put(K2, V2, Acc) end, #{}, Schema#{<<"$id">> => Urn})}. - -format_schema_field('$$ref', SubPath, PSN, PSV, #{urn := Urn}) -> - [SN, SV] = binary:split(SubPath, <<":">>), - NewUrn = binary:replace(Urn, <>, <>), - {'$ref', NewUrn}; -format_schema_field(Key, Map, PSn, PSv, XUrn) when is_map(Map) -> - {Key, maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, PSn, PSv, XUrn), maps:put(K2, V2, Acc) end, #{}, Map)}; -format_schema_field(Key, List, PSn, PSv, XUrn) when is_list(List) -> - {Key, lists:map(fun (V) -> format_schema_field(V, PSn, PSv, XUrn) end, List)}; -format_schema_field(K, V, _, _, _) -> - {K, V}. -format_schema_field(Map, PSn, PSv, XUrn) when is_map(Map) -> - maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, PSn, PSv, XUrn), maps:put(K2, V2, Acc) end, #{}, Map); -format_schema_field(List, PSn, PSv, XUrn) when is_list(List) -> - lists:map(fun (V) -> {_, Va} = format_schema_field(V, PSn, PSv, XUrn), Va end, List); -format_schema_field(Value, _, _, _) -> - Value. - -schema_to_urn(NameAndVer, XUrn) -> - [Name, Ver] = binary:split(NameAndVer, <<":">>), - Namespace = case XUrn of - #{resource := #{namespace := NS}} -> NS; - #{resource := #{directory := #{namespace := NS}}} -> NS; - #{resource := #{resource := #{namespace := NS}}} -> NS - end, - dreki_urn:to_urn(XUrn#{resource => #{namespace => NS, schema => #{schema => Name, version => Ver}}}). - -validate(XUrn, Data) -> - Res = case maps:get(<<"@schema">>, Data, undefined) of - undefined -> get(XUrn#{resource => #{namespace => get_xurn_namespace(XUrn), schema => #{schema => default}}}); - Urn = <<"dreki:", _/binary>> -> get(Urn); - SchemaNameAndVer -> get(schema_to_urn(SchemaNameAndVer, XUrn)) - end, - case Res of - {ok, Schema} -> validate_(XUrn, Schema, Data); - Error -> Error - end. - -get_xurn_namespace(#{resource := #{namespace := NS}}) -> NS; -get_xurn_namespace(#{resource := #{directory := #{namespace := NS}}}) -> NS; -get_xurn_namespace(#{resource := #{resource := #{namespace := NS}}}) -> NS. - -validate_(XUrn, Schema, Data) -> - JesseOpts = [{allowed_errors, infinity}, - {schema_loader_fun, fun get/1}], - jesse:validate_with_schema(Schema, Data, JesseOpts). diff --git a/apps/dreki/src/dreki_store_backend.erl b/apps/dreki/src/dreki_store_backend.erl deleted file mode 100644 index 55db90e..0000000 --- a/apps/dreki/src/dreki_store_backend.erl +++ /dev/null @@ -1,33 +0,0 @@ --module(dreki_store_backend). --include("dreki.hrl"). - --type t() :: module(). - --type backend_ref() :: any(). --type backend_checkout_ref() :: any(). - --type args() :: any(). - --callback valid_store(dreki_expanded_uri(), Namespace_mod :: module(), Args :: any()) -> ok | {error, any()}. - --callback start() -> ok | {error, ba}. - --callback start(binary(), binary(), dreki_urn:urn(), args()) -> {ok, backend_ref()} | {error, any()}. - --callback stop() -> ok. - --callback stop(backend_ref()) -> ok. - --callback checkout(backend_ref()) -> {ok, backend_checkout_ref()} | {error, any()}. - --callback checkin(backend_checkout_ref()) -> ok. - --callback list(backend_checkout_ref()) -> {ok, dreki_store_namespace:collection()}. - --callback get(backend_checkout_ref(), dreki_id()) -> {ok, dreki_store_namespace:item()} | not_found | {error, any()}. - --callback create(backend_checkout_ref(), dreki_store_namespace:item()) -> ok | {error, any()}. - --callback update(backend_checkout_ref(), dreki_store_namespace:item()) -> ok | not_found | {error, any()}. - --callback delete(backend_checkout_ref(), dreki_id()) -> ok | not_found | {error, any()}. diff --git a/apps/dreki/src/dreki_store_namespace.erl b/apps/dreki/src/dreki_store_namespace.erl deleted file mode 100644 index 3095ef7..0000000 --- a/apps/dreki/src/dreki_store_namespace.erl +++ /dev/null @@ -1,14 +0,0 @@ --module(dreki_store_namespace). --include("dreki.hrl"). - --type t() :: module(). - --type item() :: any(). --type collection() :: [item()]. --type name() :: binary(). - --callback start() -> ok | {error, any()}. --callback format_item(item()) -> ok | {ok, item()} | ignore. --callback valid_store(name(), Location :: dreki_urn:urn(), StoreName :: binary(), BackendModule :: module()) -> ok | {error, any()}. --callback version() -> non_neg_integer(). --callback schemas() -> #{default := Id :: binary(), Id :: binary() => #{default_version := Vsn :: binary(), Vsn :: binary => Schema :: #{}}}. diff --git a/apps/dreki/src/dreki_sup.erl b/apps/dreki/src/dreki_sup.erl index c1aa636..60b33e5 100644 --- a/apps/dreki/src/dreki_sup.erl +++ b/apps/dreki/src/dreki_sup.erl @@ -16,20 +16,12 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). -%% sup_flags() = #{strategy => strategy(), % optional -%% intensity => non_neg_integer(), % optional -%% period => pos_integer()} % optional -%% child_spec() = #{id => child_id(), % mandatory -%% start => mfargs(), % mandatory -%% restart => restart(), % optional -%% shutdown => shutdown(), % optional -%% type => worker(), % optional -%% modules => modules()} % optional init([]) -> SupFlags = #{strategy => one_for_all, intensity => 10, period => 5}, ChildSpecs = [ + #{id => drekid, start => {drekid, start_link, [[]]}}, #{id => dreki_event_manager, start => {dreki_event_manager, start_link, [[]]}}, #{id => dreki_world_server, start => {dreki_world_server, start_link, [[]]}}, #{id => dreki_node_server, start => {dreki_node_server, start_link, [[]]}} diff --git a/apps/dreki/src/dreki_task.erl b/apps/dreki/src/dreki_task.erl deleted file mode 100644 index b762aea..0000000 --- a/apps/dreki/src/dreki_task.erl +++ /dev/null @@ -1,58 +0,0 @@ --module(dreki_task). - --include("dreki.hrl"). - --type new_params() :: #{handler := dreki_task_handler(), - id => dreki_id(), - description => binary(), - params => #{}}. - --export([new/1, validate/1, to_map/1]). --export([id/1, description/1, handler/1, params/1]). --export([description/2, handler/2, params/2]). - --spec new(new_params()) -> {ok, dreki_task()} | {error, Reason::term()}. -new(Map) -> - Id = maps:get(id, Map, dreki_id:get()), - Description = maps:get(description, Map, undefined), - Params = maps:get(params, Map, #{}), - Handler = maps:get(handler, Map, undefined), - Task = #dreki_task{id=Id, handler=Handler, params=Params, description=Description, - persisted=false, dirty=true}, - validate(Task). - --spec validate(dreki_task()) -> {ok, dreki_task()} | {error, Reason::term()}. -validate(#dreki_task{handler = undefined}) -> - {error, {required, handler}}; -validate(Task = #dreki_task{id = Id, handler = _Handler}) -> - case dreki_id:valid(Id) of - {error, Err} -> {error, Err}; - ok -> {ok, Task} - end. - -id(Task = #dreki_task{}) -> - Task#dreki_task.id. - -description(Task = #dreki_task{}) -> - Task#dreki_task.description. - -handler(Task = #dreki_task{}) -> - Task#dreki_task.handler. - -params(Task = #dreki_task{}) -> - Task#dreki_task.params. - -handler(Task = #dreki_task{}, NewHandler) -> - Task#dreki_task{handler=NewHandler, dirty=true}. - -description(Task = #dreki_task{}, NewDescription) -> - Task#dreki_task{description=NewDescription, dirty=true}. - -params(Task = #dreki_task{}, NewParams) -> - Task#dreki_task{params=NewParams, dirty=true}. - -to_map(Task = #dreki_task{id = Id, handler = Handler, description = Description, params = Params}) -> - #{id => Id, - handler => Handler, - description => Description, - params => Params}. diff --git a/apps/dreki/src/dreki_tasks.erl b/apps/dreki/src/dreki_tasks.erl deleted file mode 100644 index 0899386..0000000 --- a/apps/dreki/src/dreki_tasks.erl +++ /dev/null @@ -1,141 +0,0 @@ --module(dreki_tasks). --include("dreki.hrl"). - --behaviour(dreki_store_namespace). --export([start/0, version/0, valid_store/4, format_item/1, schemas/0, new/0]). - -%% old stuff --export([resolve/1, exists/1, read_uri/2]). - -start() -> ok. - -valid_store(_Namespace, _Location, _Name, _BackendMod) -> ok. - -format_item(Item) -> ok. - -handlers() -> [dreki_tasks_script, dreki_tasks_cloyster]. - --record(?MODULE, { - id, - version, - schema, - handler, - handler_manifest -}). - -version() -> 1. - -new() -> - #{ - <<"@schema">> => <<"task:1.0">>, - <<"id">> => ksuid:gen_id(), - <<"handler">> => <<"dreki_tasks_cloyster">>, - <<"handler_manifest">> => #{ - <<"@schema">> => <<"cloyster-task:1.0">>, - <<"script">> => <<>> - } - }. - -schemas() -> - Subs = lists:foldr(fun (Handler, Acc) -> maps:merge(Acc, Handler:schemas()) end, - #{}, handlers()), - maps:merge(Subs, #{ - default => <<"task">>, - <<"task">> => schemas(task) - }). - -schemas(task) -> - #{ - default_version => <<"1.0">>, - <<"1.0">> => schemas(task, <<"1.0">>) - }. - -schemas(task, <<"1.0">>) -> - Handlers = handlers(), - Manifests0 = lists:map(fun (Handler) -> {Handler, Handler:schema_field(handler_manifest)} end, Handlers), - Manifests = lists:foldr(fun - ({Handler, undefined}, Acc) -> Acc; - ({Handler, Schema}, Acc) -> - Schemas = maps:fold(fun - (Atom, _, Acc) when is_atom(Atom) -> Acc; - (Vsn, _, Acc) -> [#{'$$ref' => <>} | Acc] - end, [], maps:get(Schema, Handler:schemas())), - Acc ++ Schemas - end, [], Manifests0), - - #{ - version => 'draft-06', - title => <<"Task">>, - type => object, - properties => #{ - id => #{type => string, <<"dreki:form">> => #{default => {ksuid, gen_id, []}}}, - schema => #{type => string}, - name => #{type => string, title => <<"Name">>}, - handler => #{type => string, title => <<"Handler">>, enum => handlers()}, - handler_manifest => #{'$ref' => <<"handler_manifest">>} - }, - '$defs' => #{ - <<"handler_manifest">> => #{anyOf => Manifests} - }, - required => [handler, handler_manifest] - }. - -%% old stuff - -read_uri(undefined, Uri) -> - {ok, #{stores => #{}}}; -read_uri(Path, Uri) -> - case binary:split(Path, <<":">>, [global]) of - [<<>>, <<>>] -> {error, invalid_resource}; - [<<>>, Id] -> {ok, #{resolve => #{kind => tasks, id => Id}}}; - [S] -> {ok, #{store => #{kind => tasks, id => S}}}; - [S, <<>>] -> {ok, #{store => #{kind => tasks, id => S}}}; - [S, Id] -> {ok, #{resource => #{kind => tasks, store => S, id => Id}}}; - R -> {error, {invalid_address, {Path, R}}} - end. - -all(Uri) when is_binary(Uri) -> - all(dreki_world:read_uri(Uri)); -all({ok, Uri = #{kind := tasks, uri := Uri, resource := Res = #{store := #{id := _S}}}}) -> - {ok, Mod, Store} = dreki_node:get_store(Uri), - Mod:all(Store); -all({ok, Uri}) -> {error, unresolvable_uri, Uri}; -all(Error = {error, _}) -> Error. - - -resolve(Id) -> - Path = dreki_world:path(), - case binary:match(Id, <>) of - {0,End} -> - StoreAndId = binary:part(Id, {End, byte_size(Id) - End}), - [StoreN, LId] = binary:split(StoreAndId, <<":">>), - {ok, {node(), StoreN, LId}} - end. - -exists({Node, StoreN, LId}) when Node =:= node() -> - #{mod := Mod, args := Args} = maps:get(StoreN, load_local_stores()), - {ok, Db} = Mod:open(Args), - Mod:exists(Db, LId); -exists(N = {Node, _, _}) -> - erpc:call(Node, dreki_tasks, exists, [N]); -exists(Id) when is_binary(Id) -> - case resolve(Id) of - {ok, N} -> exists(N); - Error = {error, _} -> Error - end. - -load_local_stores() -> - {ok, Val} = application:get_env(dreki, local_tasks_stores), - _World = #{path := Path, me := Me} = dreki_world:to_map(), - MapFn = fun ({Name, Mod, Args, Env}, Acc) -> - Store = #{local => true, - node => Me, - path => <>, - mod => Mod, - args => Args, - env => Env, - name => Name - }, - maps:put(Name, Store, Acc) - end, - lists:foldr(MapFn, #{}, Val). diff --git a/apps/dreki/src/dreki_tasks_cloyster.erl b/apps/dreki/src/dreki_tasks_cloyster.erl deleted file mode 100644 index 3fb045d..0000000 --- a/apps/dreki/src/dreki_tasks_cloyster.erl +++ /dev/null @@ -1,21 +0,0 @@ --module(dreki_tasks_cloyster). --export([schemas/0, schema_field/1]). - -schema_field(handler_manifest) -> <<"cloyster-task">>. - -schemas() -> - #{ - <<"cloyster-task">> => #{ - default_version => <<"1.0">>, - <<"1.0">> => #{ - version => 'draft-06', - title => <<"Cloyster Task Definition">>, - type => object, - properties => #{ - <<"script">> => #{type => string} - }, - required => [script] - } - } - }. - diff --git a/apps/dreki/src/dreki_tasks_script.erl b/apps/dreki/src/dreki_tasks_script.erl deleted file mode 100644 index 8eeb563..0000000 --- a/apps/dreki/src/dreki_tasks_script.erl +++ /dev/null @@ -1,24 +0,0 @@ --module(dreki_tasks_script). --export([schemas/0, schema_field/1]). - -schema_field(handler_manifest) -> <<"script-task">>. - -schemas() -> - #{ - <<"script-task">> => #{ - default_version => <<"1.0">>, - <<"1.0">> => #{ - version => 'draft-06', - title => <<"Executable Script Task">>, - type => object, - properties => #{ - <<"id">> => #{type => string, <<"dreki:form">> => #{default => generate_id}}, - <<"name">> => #{type => string}, - <<"description">> => #{type => string, <<"dreki:form">> => #{input => textarea, textarea_mode => markdown}}, - <<"executable">> => #{type => string, default => <<"/bin/sh">>}, - <<"script">> => #{type => string, <<"dreki:form">> => #{input => textarea}} - }, - required => [script] - } - } - }. diff --git a/apps/dreki/src/dreki_world.erl b/apps/dreki/src/dreki_world.erl deleted file mode 100644 index 437b6c8..0000000 --- a/apps/dreki/src/dreki_world.erl +++ /dev/null @@ -1,366 +0,0 @@ --module(dreki_world). - --export([ensure_consistency/0, index/2]). --export([get_path/1, get_path/2, paths/0, put_path/3]). --export([get_region_from_dns_name/1, get_region/1, get_region/2, get_region/3, create_region/1, put_region/3]). --export([node/0, nodes/0, get_node_from_dns_name/1, get_node/1, create_node_from_dns_name/2, create_node/2, put_node/3]). --export([refresh_index/2, get/2, get/3, maybe_put_index/5]). --export([namespace_to_module/1]). --export([path_to_domain/1]). --export([to_map/0, root_domain/0, internal_domain/0, domain/0, hierarchy/0, parent/0, me/0, path/0, root_path/0, strip_root_path/1, put_root_path/1, read_uri/1, domain_to_path/1]). - -ensure_consistency() -> - Dns = dreki_world_dns:as_map(), - Vertices = maps:get(vertices, Dns), - ensure_consistency(Vertices, []). - -ensure_consistency([#{type := root, name := Root} | Rest], Acc) -> - Acc = case get_region_from_dns_name(Root) of - {error, {not_found, _}} -> - ok = create_region_from_dns_name(Root), - put_region(Root, root, true), - [{root, Root} | Acc]; - _ -> - Acc - end, - Res = [Log || Kind <- [tasks], Log = {create, _} <- [ensure_global_root_stores(Root, Kind)]], - ensure_consistency(Rest, Res ++ Acc); - -ensure_consistency([#{type := region, name := Region} | Rest], Acc) -> - case get_region_from_dns_name(Region) of - {error, {not_found, _}} -> - ensure_consistency(Rest, [{Region, create_region_from_dns_name(Region)} | Acc]); - _ -> - ensure_consistency(Rest, Acc) - end; -ensure_consistency([_ | Rest], Acc) -> - ensure_consistency(Rest, Acc); -ensure_consistency([], Acc) -> - Acc. - -ensure_global_root_stores(Root, Kind) -> - {ok, Region} = get_region_from_dns_name(Root), - RegionUri = maps:get(uri, Region), - KindB = atom_to_binary(Kind), - StoreUri = <>, - case dreki_store:get_store(StoreUri) of - {ok, _store} -> ok; - _ -> {create, {Root,Kind,dreki_store:create_store(StoreUri, dreki_world_store, #{}, #{})}} - end. - -put_index(IdxKey, IdxValue, Uri, Data) -> - plum_db:put({IdxKey, IdxValue}, Uri, Data). - -maybe_put_index(Kind, IdxKey, IdxValue, Uri, Data) -> - case lists:member(IdxKey, index_fields(Kind)) of - true -> put_index(index_key(IdxKey), IdxValue, Uri, Data); - false -> ok - end. - -index_key(roles) -> 'idx:roles'; -index_key(tags) -> 'idx:tags'. - -index(Key, Value) -> - Idx = index_key(Key), - plum_db:fold(fun - ({_, ['$deleted']}, Acc) -> Acc; - ({Key, Val}, Acc) -> - [{Key, Val} | Acc] - end, [], {Idx, Value}). - -index_fields(node) -> [roles, tags]; -index_fields(region) -> [roles, tags]; -index_fields(_) -> [roles, tags]. - -refresh_index(Kind, Map) when is_atom(Kind) -> - refresh_index(index_fields(Kind), Kind, Map). -refresh_index([Field | Rest], Kind, Map = #{uri := URI}) -> - Values = case maps:get(Field, Map, undefined) of - undefined -> []; - V when is_list(V) -> V; - V -> [V] - end, - [maybe_put_index(Kind, Field, V, URI, undefined) || V <- Values], - refresh_index(Rest, Kind, Map); -refresh_index([], _, _) -> - ok. - -clean_path(Path) -> - Len = byte_size(Path) - 1, - case Path of - <> -> P; - P -> P - end. - -put_path(Path0, Kind, Uri) -> - Path = clean_path(Path0), - {ok, Links} = get({paths, Path}, Path, [{default, #{}}]), - case maps:find(Kind, Links) of - {ok, _} -> {error, {exists, {Path, Kind}}}; - error -> - ok = plum_db:put({paths, Path}, Path, Links#{Kind => Uri}), - ok - end. - -get_path(Path0) -> - Path = clean_path(Path0), - case get({paths, Path}, Path) of - not_found -> {error, {not_found, {path, Path}}}; - {ok, Data} -> {ok, Data} - end. - -get_path(Path, Kind) -> - case get_path(Path) of - {ok, Map} -> - case maps:get(Kind, Map, undefined) of - undefined -> {error, {not_found, {Path, Kind}}}; - Value -> {ok, Value} - end; - Error = {error, _} -> Error - end. - -paths() -> - plum_db:fold(fun - ({_, ['$deleted']}, Acc) -> Acc; - ({Path, Value}, Acc) -> - maps:put(Path, Value, Acc) - end, #{}, {paths, '_'}). - -get(Prefix, Key) -> - get(Prefix, Key, []). - -get(Prefix, Key, Opts) -> - case plum_db:get(Prefix, Key, Opts) of - undefined -> not_found; - ['$deleted'] -> not_found; - Val -> {ok, Val} - end. - -region_uri_from_dns_name(Name) -> - Root = internal_domain(), - if - Root =:= Name -> root_path(); - true -> domain_to_path(Name) - end. - -get_region_from_dns_name(Name) -> - get_region(region_uri_from_dns_name(Name)). - -get_region(Path) -> - case get_path(Path, region) of - {ok, _} -> - Region = plum_db:fold(fun - ({_, ['$deleted']}, M) -> M; - ({{_, K}, V}, M) -> maps:put(K, V, M) - end, #{uri => Path}, {regions, Path}), - {ok, Region}; - Error -> Error - end. - -get_region(Path, Key) -> - get_region(Path, Key, undefined). - -get_region(Path, Key, GetOpts) -> - case get_path(Path, region) of - {ok, _} -> get({regions, Path}, {Path, Key}, GetOpts); - Error -> Error - end. - -create_region_from_dns_name(Name) -> - create_region(region_uri_from_dns_name(Name)). - -create_region(Path) -> - case get_region(Path) of - {ok, _} -> {error, {exists, {region, Path}}}; - {error, {not_found, _}} -> - ok = put_path(Path, region, Path), - ok = put_region(Path, region, Path), - {ok, R} = get_region(Path), - ok = refresh_index(region, R), - ok - end. - -put_region(Path, Key, Value) -> - case get_path(Path, region) of - {ok, _} -> - ok = plum_db:put({regions, Path}, {Path, Key}, Value), - ok = maybe_put_index(region, Key, Value, Path, undefined); - Error -> Error - end. - -nodes() -> - lists:foldr(fun (Domain, Acc) -> - case get_node_from_dns_name(Domain) of - {ok, Node} -> [Node | Acc]; - _ -> Acc - end - end, [], dreki_world_dns:nodes()). - -node() -> - dreki_world:domain_to_path(dreki_world:domain()). - -get_node_from_dns_name(Name) -> - get_node(domain_to_path(Name)). - -get_node(Path) -> - case get_path(Path, node) of - {ok, _} -> - Node = plum_db:fold(fun - ({_, ['$deleted']}, M) -> M; - ({{_, K}, [V]}, M) -> maps:put(K, V, M) - end, #{uri => Path}, {nodes, Path}), - {ok, Node}; - Error -> Error - end. - -create_node_from_dns_name(Name, Params) -> - create_node(domain_to_path(Name), Params). - -create_node(Path, Params) -> - case get_path(Path, node) of - {ok, _} -> {error, {exists, {node, Path}}}; - {error, {not_found, _}} -> - ok = put_path(Path, node, Path), - ok = put_node(Path, node, Path), - [ok = put_node(Path, Key, Value) || {Key, Value} <- maps:to_list(Params)], - ok - end. - -put_node(Path, Key, Value) -> - case get_path(Path, node) of - {ok, _} -> plum_db:put({nodes, Path}, {Path, Key}, Value); - Error -> Error - end. - -domain_to_path(Domain) -> - Clean = strip_domain(Domain, internal_domain()), - Parts = binary:split(Clean, <<".">>, [global]), - Components = [<<"dreki">>, root_domain(), internal_subdomain()] ++ lists:reverse(Parts), - join(<<":">>, Components). - -path_to_domain(<<"dreki:", Rest/binary>>) -> - [Location | _] = binary:split(Rest, <<"::">>), - Components = binary:split(Location, <<":">>, [global]), - join(<<".">>, lists:reverse(Components)). - -root_domain() -> - get_env(root_domain). - -internal_domain() -> - get_env(internal_domain). - -domain() -> - get_env(domain). - -read_uri(U = <<"dreki:", _/binary>>) -> - read_uri(U, root_path()); -read_uri(Invalid) -> - {error, {bad_type, Invalid}}. -%% Remove root -%%read_uri(U, Root) -> -%% read_uri(U, Root, Rest). -%%read_uri(U, Root) -> -%% {error, #{message => <<"URI outside of domain">>, uri => U, domain => Root}}. -%% Extract hierarchy -read_uri(U, Root) -> - {Hier, Res} = case binary:split(U, <<"::">>) of - [H, R] -> {H, R}; - [H] -> {H, undefined} - end, - case get_path(H) of - {ok, Thing} -> read_uri_(U, Root, H, Res, Thing); - Error -> Error - end. - -read_uri_(U, Root, Path, ResPath, Thing) -> - Kind = case maps:keys(Thing) of - [K] -> K; - Ks -> Ks - end, - RegionStoreHint = case {maps:get(region, Thing, false), maps:get(node, Thing, false)} of - {false, _} -> global; - {_, false} -> global; - {_, _} -> local - end, - StoreHints = lists:foldr(fun - (node, Acc) -> maps:put(node, local, Acc); - (region, Acc) -> maps:put(region, RegionStoreHint, Acc) - end, #{}, maps:keys(Thing)), - Uri = #{domain => Root, path => Path, kind => Kind, store_hints => StoreHints, uri => U}, - case read_resource_uri(ResPath, Uri) of - {ok, Resource} -> - {ok, Uri#{resource => Resource}}; - {error, invalid_resource} -> - {error, #{message => <<"Invalid resource">>, uri => U, resource => ResPath}}; - {error, invalid_namespace, NS} -> - {error, #{message => <<"Invalid namespace">>, uri => U, namespace => NS}} - end. - -read_resource_uri(undefined, _) -> - {ok, undefined}; -read_resource_uri(Path, Uri) -> - {NS, Rest} = case binary:split(Path, <<":">>) of - [NS, R] -> {NS, R}; - [NS] -> {NS, undefined} - end, - case namespace_to_module(NS) of - {ok, Mod} -> Mod:read_uri(Rest, Uri); - error -> {error, invalid_namespace, NS} - end. - -namespace_to_module(<<"tasks">>) -> namespace_to_module(tasks); -namespace_to_module(<<"names">>) -> namespace_to_module(names); -namespace_to_module(tasks) -> {ok, dreki_tasks}; -namespace_to_module(names) -> {ok, dreki_names}; -namespace_to_module(_) -> error. - -internal_subdomain() -> - strip_domain(internal_domain(), root_domain()). - -root_path() -> - join(<<":">>, [<<"dreki">>, root_domain(), internal_subdomain()]). - -strip_root_path(Path) -> - Root = root_path(), - binary:replace(Path, <>, <<"">>). - -put_root_path(Path) -> - join(<<":">>, [root_path(), Path]). - -path() -> - HierJ = lists:reverse(hierarchy()), - Components = [<<"dreki">>, root_domain(), internal_subdomain()] ++ HierJ, - join(<<":">>, Components). - -hierarchy() -> - binary:split(strip_domain(domain(), internal_domain()), <<".">>, [global]). - -parent() -> - case hierarchy() of - [_, Parent | _] -> Parent; - [Parent] -> Parent - end. - -me() -> - [Me | _] = hierarchy(), - Me. - -get_env(Key) -> - {ok, Val} = application:get_env(dreki, Key), - Val. - -to_map() -> - #{domain => #{root => root_domain(), internal => internal_domain(), self => domain()}, - path => path(), - root_path => root_path(), - hiearchy => lists:reverse(hierarchy()), - parent => parent(), - me => me()}. - -strip_domain(FQDN, Parent) -> - binary:replace(FQDN, <<".", Parent/binary>>, <<"">>). - -join(_Separator, []) -> - <<>>; -join(Separator, [H|T]) -> - lists:foldl(fun (Value, Acc) -> <> end, H, T). diff --git a/apps/dreki/src/dreki_world_dns.erl b/apps/dreki/src/dreki_world_dns.erl deleted file mode 100644 index 058c9db..0000000 --- a/apps/dreki/src/dreki_world_dns.erl +++ /dev/null @@ -1,162 +0,0 @@ --module(dreki_world_dns). - --export([start/0, graph/0, node_ips/1, node/0, parents/1, nodes/0, regions/0, vertex/1, node_params/1, node_param/2, as_map/0]). --export([get_path/2, in_neighbours/1, out_neighbours/1]). - -get_path(V1, V2) -> - digraph:get_path(graph(), V1, V2). - -in_neighbours(V) -> - digraph:in_neighbours(graph(), V). - -out_neighbours(V) -> - digraph:out_neighbours(graph(), V). - -as_map() -> - Vertices = lists:foldr(fun (V = {Type, Key}, Acc) -> - {_, Label} = digraph:vertex(graph(), V), - BType = atom_to_binary(Type), - Node = <>, - [#{node => Node, type => Type, name => Key, data => Label} | Acc] - end, [], digraph:vertices(graph())), - Edges = lists:foldr(fun (E, Acc) -> - {_, {Ft, Fn}, {Tt, Tn}, Label} = digraph:edge(graph(), E), - BFt = atom_to_binary(Ft), - BTt = atom_to_binary(Tt), - Fk = <>, - Tk = <>, - [#{from => Fk, to => Tk, data => Label} | Acc] - end, [], digraph:edges(graph())), - #{vertices => Vertices, edges => Edges}. - -vertex(V) -> - digraph:vertex(graph(), V). - -nodes() -> - [N || V = {node, N} <- digraph_utils:topsort(graph())]. - -regions() -> - vertices(region). - -vertices(Type) -> - [V || V = {Type, _} <- digraph_utils:topsort(graph())]. - -node() -> - {node, dreki_world:domain()}. - -node_params(N) -> - case digraph:vertex(graph(), {node, N}) of - {_, Params} -> {ok, Params}; - _ -> {error, no_such_node} - end. - -node_param(N, Key) -> - case node_params(N) of - {ok, P} -> {ok, maps:get(Key, P)}; - Err -> Err - end. - -parents(V) -> - digraph:in_neighbours(graph(), V). - -node_ips(N) -> - case digraph:vertex(graph(), {node, N}) of - {{node, N}, #{srvs := SRVs}} -> - IPs = [ {I,Port} || #{name := Name, port := Port} <- SRVs, - T <- [a, aaaa], - {ok, {_,_,_,_,_,Ip}} <- [inet_res:getbyname(binary_to_list(Name), T)], - I <- Ip], - {ok, lists:flatten(IPs)}; - Err -> - {error, {no_such_node, N, Err}} - end. - -start() -> - {ok, Graph, Errs} = build(), - persistent_term:put({?MODULE, graph}, Graph), - {ok, Errs}. - -graph() -> - persistent_term:get({?MODULE, graph}). - -build() -> - Root = dreki_world:internal_domain(), - Host = sd_dns(Root), - case inet_res:getbyname(Host, srv) of - {ok, {hostent, _Host, [], srv, _, SRVs}} -> - Graph = digraph:new([acyclic]), - {Name, NameErrs} = read_txt(name_dns(Root), Root), - V = {root, Root}, - digraph:add_vertex(Graph, V, #{display_name => Name}), - Errs = collect_sd_srvs(SRVs, V, Graph, NameErrs), - {ok, Graph, lists:flatten(Errs)}; - {error, DNSErr} when is_atom(DNSErr) -> - {error, #{error => "world_dns_error", dns_error => DNSErr, host => Host}} - end. - -expand_sd_srv(Host, Parent, Graph) -> - NodeHost = node_dns(Host), - {Vn, Acc0} = case inet_res:getbyname(NodeHost, srv) of - {ok, {hostent, _, _, srv, _, NSRVs}} -> - Targets = lists:foldr(fun ({Priority, Weight, Port, Name}, Acc) -> - [#{name => list_to_binary(Name), port => Port, priority => Priority, weight => Weight} | Acc] - end, [], NSRVs), - {NName, NameErrs} = read_txt(name_dns(Host), name(Host)), - {NodeName, NameErrs2} = read_txt(node_name_dns(Host), <<"dreki@", Host/binary>>), - Nv = {node, Host}, - digraph:add_vertex(Graph, Nv, #{display_name => NName, srvs => Targets, node_name => binary_to_atom(NodeName)}), - digraph:add_edge(Graph, Parent, Nv, #{}), - {Nv, [] ++ NameErrs ++ NameErrs2}; - {error, nxdomain} -> {undefined, []}; - {error, VDNSErr} when is_atom(VDNSErr) -> - logger:log(error, #{dns_error => VDNSErr, host => NodeHost}), - {undefined, [{error, #{error => "world_dns_error", dns_error => VDNSErr, host => NodeHost}}]} - end, - SdHost = sd_dns(Host), - Acc1 = case inet_res:getbyname(SdHost, srv) of - {ok, {hostent, _SdHost, [], srv, _, SSRVs}} -> - {RName, RNameErrs} = read_txt(name_dns(Host), name(Host)), - V = {region, Host}, - digraph:add_vertex(Graph, V, #{display_name => RName}), - case Vn of - undefined -> digraph:add_edge(Graph, Parent, V, #{}); - Vn -> digraph:add_edge(Graph, Vn, V, #{}) - end, - collect_sd_srvs(SSRVs, V, Graph, RNameErrs); - {error, nxdomain} -> []; - {error, DNSErr} when is_atom(DNSErr) -> - logger:log(error, #{dns_error => DNSErr, host => SdHost}), - [{error, #{error => "world_dns_error", dns_error => DNSErr, host => SdHost}}] - end, - [Acc0, Acc1]. - -collect_sd_srvs([], _, _Graph, Acc) -> Acc; -collect_sd_srvs([{0, 0, 1337, Entry} | Rest], Parent, Graph, Acc) -> - collect_sd_srvs(Rest, Parent, Graph, [expand_sd_srv(list_to_binary(Entry), Parent, Graph) | Acc]). - -read_txt(Host, Default) -> - case inet_res:getbyname(Host, txt) of - {error, nxdomain} -> {Default, []}; - {ok,{hostent, _, _, _, _, Lines}} -> {list_to_binary(Lines), []}; - {error, DNSErr} when is_atom(DNSErr) -> - logger:log(error, #{dns_error => DNSErr, host => Host}), - {Default, [#{error => "world_dns_error", dns_error => DNSErr, host => Host}]} - end. - -sd_dns(Domain) -> dnsname(<<"_dreki">>, Domain). -node_dns(Domain) -> dnsname(<<"_node._dreki">>, Domain). -name_dns(Domain) -> dnsname(<<"_name._dreki">>, Domain). -node_name_dns(Domain) -> dnsname(<<"_name._node._dreki">>, Domain). - -dnsname(Prefix, Domain) when is_list(Prefix) -> - dnsname(list_to_binary(Prefix), Domain); -dnsname(Prefix, Domain) when is_list(Domain) -> - dnsname(Prefix, list_to_binary(Domain)); -dnsname(Prefix, Domain) -> - Full = <>, - binary:bin_to_list(Full). - -name(Host) when is_list(Host) -> - name(list_to_binary(Host)); -name(Host) -> - hd(binary:split(Host, <<".">>)). diff --git a/apps/dreki/src/dreki_world_plum_events.erl b/apps/dreki/src/dreki_world_plum_events.erl deleted file mode 100644 index b32ae09..0000000 --- a/apps/dreki/src/dreki_world_plum_events.erl +++ /dev/null @@ -1,17 +0,0 @@ --module(dreki_world_plum_events). --behaviour(gen_event). --export([init/1, handle_call/2, handle_event/2, terminate/2]). - -init([Server]) -> - {ok, Server}. - -handle_call(_, Server) -> - {ok, error, Server}. - -handle_event(Event, Server) -> - logger:info("plum_event: ~p", [Event]), - dreki_world_server:send_event(Server, {plum_events, Event}), - {ok, Server}. - -terminate(_, Server) -> - ok. diff --git a/apps/dreki/src/dreki_world_server.erl b/apps/dreki/src/dreki_world_server.erl deleted file mode 100644 index 2bc41ed..0000000 --- a/apps/dreki/src/dreki_world_server.erl +++ /dev/null @@ -1,63 +0,0 @@ --module(dreki_world_server). --behaviour(partisan_gen_fsm). --include_lib("kernel/include/logger.hrl"). - --export([start_link/1, send_event/2]). --export([init/1]). --export([setup/2]). --export([wait/2]). - --record(data, { - path=undefined -}). - -start_link(Args) -> - partisan_gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). - -send_event(Name, Event) -> - partisan_gen_fsm:send_event(Name, Event). - -init(Args) -> - ok = plum_db_events:add_handler(dreki_world_plum_events, [?MODULE]), - Data = #data{path = dreki_world:path()}, - ok = setup_autojoin(), - todo = setup_connect_nearest(), - ok = dreki_node:ensure_local_node(), - {ok, setup, Data, 0}. - -setup(timeout, Data) -> - ?LOG_INFO("world_server: setup done"), - {next_state, wait, Data}. - -setup_autojoin() -> - case application:get_env(dreki, autojoin, undefined) of - undefined -> ok; - List -> [dreki_peer_service:join(N) || N <- List] - end, - ok. - -setup_connect_nearest() -> - case dreki_peer_service:connect_nearest_dns() of - ok -> ok; - todo -> todo; - error -> - logger:error("Node did not join anyone from the world!!"), - error - end. - -wait({plum_events, {object_update, {Prefix, Key}, Object}}, Data) -> - process_object_update(Prefix, Key, Object, Data), - {next_state, wait, Data}; -wait(Event, Data) -> - ?LOG_INFO("world_server wait event: ~p", [Event]), - {next_state, wait, Data}. - -process_object_update({nodes, Node}, Key, _Obj, #data{path = Node}) -> - ?LOG_INFO("My node has been updated !"), - ok; -process_object_update({nodes, Node}, Key, _Obj, _Data) -> - ?LOG_INFO("Node update ~p ~p", [Node, Key]), - ok; -process_object_update(Prefix, Key, _Obj, _Data) -> - ?LOG_INFO("Metadata update ~p ~p", [Prefix, Key]), - ok. diff --git a/apps/dreki/src/dreki_world_store.erl b/apps/dreki/src/dreki_world_store.erl deleted file mode 100644 index 71ef2ce..0000000 --- a/apps/dreki/src/dreki_world_store.erl +++ /dev/null @@ -1,48 +0,0 @@ --module(dreki_world_store). --include("dreki.hrl"). --behaviour(dradis_store_backend). - -%% Types - --record(store, {}). --type db() :: #store{}. --type args() :: #{}. - --export([start/0, start/4, checkout/1, checkin/1, stop/0, stop/1]). --export([valid_store/5]). --export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). - -valid_store(_Namespace, Location, _Name, _NSMod, _Args) -> - case Location =:= dreki_world:root_path() of - true -> ok; - false -> {error, {dreki_world_store_invalid_location, Location, dreki_world:root_path()}} - end. - -start() -> ok. -start(_, _, _, _) -> {ok, dreki_world_store}. - -checkout(_) -> {ok, #store{}}. -checkin(_) -> ok. -stop() -> ok. -stop(_) -> ok. - -list(_) -> - {error, not_implemented}. - -count(_) -> - {error, not_implemented}. - -exists(_, _) -> - {error, not_implemented}. - -get(_, _) -> - {error, not_implemented}. - -delete(_, _) -> - {error, not_implemented}. - -create(_, _) -> - {error, not_implemented}. - -update(_Tab, _) -> - {error, not_implemented}. diff --git a/apps/dreki/src/dreki_world_tasks.erl b/apps/dreki/src/dreki_world_tasks.erl deleted file mode 100644 index c27a7dc..0000000 --- a/apps/dreki/src/dreki_world_tasks.erl +++ /dev/null @@ -1,66 +0,0 @@ --module(dreki_world_tasks). --include("dreki.hrl"). --define(PREFIX, {objects, 'dreki_world_tasks'}). - -%% Types --type db() :: term(). --type args() :: #{db_name => binary()}. - -%% storage-specific --export([start/1, open/1, sync/1, close/1, stop/1]). --export([all/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). - -start(_) -> {ok, dreki_world_tasks}. -open(_) -> {ok, dreki_world_tasks}. -close(_) -> ok. -stop(_) -> ok. -sync(_) -> noop. - -all(_) -> - Results = plum_db:fold(fun ({_, Value}, Acc) -> - [Value | Acc] - end, [], ?PREFIX), - {ok, Results}. - -count(_) -> - Results = plum_db:fold(fun (_, Acc) -> - Acc + 1 - end, 0, {objects, 'dreki_world_tasks'}), - {ok, Results}. - -exists(T, Id) -> - case get(T, Id) of - {ok, _} -> true; - _ -> false - end. - -get(_, Id) -> - case dreki_world:get(?PREFIX, Id) of - {ok, Val} -> {ok, Val}; - not_found -> {error, {task_not_found, Id}} - end. - -create(T, Task = #dreki_task{id = Id}) -> - case get(T, Id) of - {ok, _} -> {error, {exists, Id}}; - {error, {task_not_found, _}} -> - ok = plum_db:put(?PREFIX, Id, T), - ok = dreki_world:refresh_index(Task), - get(T, Id) - end. - -update(T, Task = #dreki_task{id = Id}) -> - case get(T, Id) of - Error = {error, {task_not_found, _}} -> Error; - {ok, _OldTask} -> - ok = plum_db:put(?PREFIX, Id, T), - ok = dreki_world:refresh_index(Task), - get(T, Id) - end. - -delete(T, #dreki_task{id = Id}) -> - {error, {dreki_world_tasks, delete_not_implemented}}. - - -refresh_index(#dreki_task{uri = URI, tags = Tags, roles = Roles}) -> - dreki_world:refresh_index(tasks, #{uri => URI, tags => Tags, roles => Roles}). diff --git a/apps/dreki/src/drekid.erl b/apps/dreki/src/drekid.erl new file mode 100644 index 0000000..73beffd --- /dev/null +++ b/apps/dreki/src/drekid.erl @@ -0,0 +1,95 @@ +-module(drekid). +-behaviour(partisan_gen_fsm). +-include_lib("kernel/include/logger.hrl"). +-include_lib("dreki_otel.hrl"). +-compile({no_auto_import, [get/0]}). +-export([get/0]). +-export([request/1, request/2, request/3]). +-export([start_link/1]). +-export([init/1, waiting/2, handle_info/3, handle_event/3, handle_sync_event/4]). +-define(DREKID_PT, drekid). +-record(drekid, {node, pid}). + +-spec get() -> {ok, pid()} | {error, drekid_unavailable}. +get() -> + persistent_term:get(drekid, {error, drekid_unavailable}). + +request(Fun) -> + request(Fun, []). + +request(Fun, Args) -> + request(Fun, Args, 5000). + +request(Fun, Args, Timeout) when is_binary(Fun) -> + ?with_span(?FUN_NAME, fun(_SpanCtx) -> + send_request(Fun, Args, Timeout) + end). + +send_request(Fun, Args, Timeout) -> + ?set_attributes(#{function => Fun, args => Args, timeout => Timeout}), + case get() of + {ok, Pid} -> + Ref = make_ref(), + logger:info("drekid_request: ~p ~p to ~p", [Fun, Args, Pid]), + Pid ! {drekid_request_v1, self(), Ref, Fun, Args, Timeout}, + ?with_span(?FUN_NAME(await_request), fun(_) -> + await_request(Ref, Timeout) + end); + Error -> + Error + end. + +await_request(Ref, Timeout) -> + receive + {drekid_response_v1, Ref, Result} -> + case Result of + {ok, Data = {_, _}} -> + Data; + {ok, Data} -> + {ok, Data}; + {error, Error} -> + {error, {drekid_error, Error}} + end + after Timeout -> + {error, {drekid_error, timeout}} + end. + +start_link(Args) -> + partisan_gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). + +%% + +init(_Args) -> + ?LOG_DEBUG("drekid initialized"), + {ok, waiting, #drekid{}, 0}. + +waiting(timeout, Data) -> + persistent_term:put(drekid, {error, drekid_unavailable}), + {next_state, waiting, Data}; +waiting({drekid, {connect, Node, Pid}}, Data) -> + Data0 = Data#drekid{node = Node, pid = Pid}, + monitor_node(Node, true), + Pid ! {drekid, hello, node(), self()}, + ?LOG_NOTICE("drekid ready, node: ~p ~p", [Node, Pid]), + persistent_term:put(drekid, {ok, Pid}), + {next_state, ready, Data}. + +down(timeout, {Reason, State, Data}) -> + ?LOG_ERROR(#{code => drekid_down, reason => Reason, state => State}), + {next_state, waiting, Data#drekid{node = undefined, pid = undefined}}. + +handle_info({drekid, Msg}, waiting, Data) -> + waiting({drekid, Msg}, Data); +handle_info({nodedown, Node}, State, Data = #drekid{node = Node}) -> + {next_state, down, {nodedown, State, Data}, 0}; +handle_info(Info, State, Data) -> + ?LOG_ERROR(#{code => unhandled_info, state => State, info => Info}), + {stop, badinfo, Data}. + +handle_event(Event, State, Data) -> + ?LOG_ERROR(#{code => unhandled_event, state => State, event => Event}), + {stop, badevent, Data}. + +handle_sync_event(Event, From, State, Data) -> + ?LOG_ERROR(#{code => unhandled_sync_event, state => State, event => Event}), + {stop, badevent, badevent, Data}. diff --git a/apps/dreki/src/drekid_tasks_store.erl b/apps/dreki/src/drekid_tasks_store.erl new file mode 100644 index 0000000..e16182a --- /dev/null +++ b/apps/dreki/src/drekid_tasks_store.erl @@ -0,0 +1,71 @@ +-module(drekid_tasks_store). +-behaviour(dreki_store_backend). + +-export([install/0]). +-export([start/0, start/5, checkout/1, checkin/1, stop/0, stop/1]). +-export([valid_store/5]). +-export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). + +tasks() -> + Tasks = #{<<"dreki.v1.exec">> => #{id => <<"dreki.v1.exec">>, module => drekid_task_handler, params => #{}}}, + FreeBSD = [<<"dreki.v1.freebsd.jails.list">>, + <<"dreki.v1.freebsd.jails.get">>, + <<"dreki.v1.freebsd.jails.start">>, + <<"dreki.v1.freebsd.jails.exec">> + ], + Tasks0 = lists:foldr(fun (Id, Acc) -> + Task = #{id => Id, module => drekid_task_handler, params => dreki_freebsd_schemas:schema(Id)}, + maps:put(Id, Task, Acc) + end, Tasks, FreeBSD), + Tasks0. + +urn() -> + Urn = dreki_node:urn(), + <>. + +install() -> + dreki_store:create_store(urn(), ?MODULE, #{}, #{}). + +valid_store(<<"tasks">>, _Loc, _Name, _NSMod, _Args) -> + ok; +valid_store(_Ns, _Loc, _Name, _NSMod, _Args) -> + error. + +start() -> + ok. + +start(_Ns, _NsMod, _Loc, _XUrn, Args) -> + {ok, Args}. + +checkout(Args) -> + {ok, Args}. + +checkin(Args) -> + ok. + +stop() -> + ok. + +stop(Args) -> + ok. + +list(_) -> + {ok, [V || {K, V} <- maps:to_list(tasks())]}. + +get(_, Id) -> + maps:get(Id, tasks(), not_found). + +count(_) -> + {ok, maps:size(tasks())}. + +exists(_, Id) -> + {ok, maps:is_key(Id, tasks())}. + +create(_, _) -> + {error, not_supported}. + +update(_, _) -> + {error, not_supported}. + +delete(_, _) -> + {error, not_supported}. diff --git a/apps/dreki/src/funs/dreki_fun_exec.erl b/apps/dreki/src/funs/dreki_fun_exec.erl new file mode 100644 index 0000000..d704e80 --- /dev/null +++ b/apps/dreki/src/funs/dreki_fun_exec.erl @@ -0,0 +1,28 @@ +-module(dreki_fun_exec). +-behaviour(dreki_funs). +-export([schemas/0]). + +schemas() -> + #{<<"exec">> => #{default_version => <<"1.0">>, <<"1.0">> => schemas('1.0')}}. + +schemas('1.0') -> + #{ + version => 'draft-06', + title => <<"Execute command">>, + type => object, + properties => #{ + <<"id">> => #{type => string, + <<"dreki:form">> => #{default => generate_id}}, + <<"name">> => #{type => string}, + <<"description">> => #{type => string, + <<"dreki:form">> => #{ + input => textarea, + textarea_mode => markdown + }}, + <<"command">> => #{type => string, <<"dreki:form">> => #{ + input_style => monospace, + placeholder => <<"/bin/true">> + }} + }, + required => [id, name, command] + }. diff --git a/apps/dreki/src/funs/dreki_funs.erl b/apps/dreki/src/funs/dreki_funs.erl new file mode 100644 index 0000000..1d15f21 --- /dev/null +++ b/apps/dreki/src/funs/dreki_funs.erl @@ -0,0 +1,31 @@ +-module(dreki_funs). +-include("dreki.hrl"). + +-behaviour(dreki_store_namespace). +-export([start/0, valid_store/4, format_item/1, schemas/0]). + +-callback(schemas() -> #{}). + +start() -> + ok. + +valid_store(_Namespace, _Location, _Name, _BackendMod) -> + ok. + +format_item(Item) -> + ok. + +handlers() -> + [dreki_fun_exec]. + +-record(?MODULE, { + id, + version, + name, + handler, + content + }). + +schemas() -> + Schemas = lists:foldr(fun (Handler, Acc) -> maps:merge(Acc, Handler:schemas()) end, #{}, handlers()), + maps:put(default, <<"exec:1.0">>, Schemas). diff --git a/apps/dreki/src/node/dreki_node.erl b/apps/dreki/src/node/dreki_node.erl new file mode 100644 index 0000000..5ed7fac --- /dev/null +++ b/apps/dreki/src/node/dreki_node.erl @@ -0,0 +1,150 @@ +-module(dreki_node). +-include("dreki.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include("dreki_otel.hrl"). +-behaviour(partisan_gen_fsm). +-compile({no_auto_import,[get/0]}). +-export([get/0, urn/0, stores/0]). +-export([rpc/4, rpc/5, remote_rpc/7]). +-export([uri/0]). % deprecated +-export([parents/0, parents/1, parent/0, parent/1]). +-export([neighbours/0, neighbours/1]). +-export([descendants/0, descendants/1]). +-export([ensure_local_node/0]). + +-type rpc_error() :: {rpc_error, dreki_urn(), timeout | any()}. + +rpc(Path, Mod, Fun, Args) -> + rpc(Path, Mod, Fun, Args, #{timeout => 1000}). + +-spec rpc(dreki_urn(), module(), function(), Args :: [], #{timeout => non_neg_integer()}) -> {ok, any()} | {error, rpc_error()}. +rpc(Path, Mod, Fun, Args, #{timeout := Timeout}) -> + Myself = self(), + ?with_span(?FUN_NAME, fun(SpanCtx) -> + ?set_attributes(#{mod => Mod, fn => Fun, remote_node_path => Path}), + ?LOG_INFO("Ctx ~p ~p ~p ~p", [?FUN_NAME, otel_ctx:get_current(), otel_tracer:current_span_ctx(), SpanCtx]), + case dreki_world_dns:node_param(dreki_world:path_to_domain(Path), node_name) of + {ok, NodeName} -> + ?set_attribute(remote_node, NodeName), + Ctx = otel_ctx:get_current(), + ChildSpanCtx = ?start_span(?FUN_NAME(remote), #{}), + TraceId = otel_span:hex_trace_id(SpanCtx), + SpanId = otel_span:hex_span_id(SpanCtx), + case partisan_rpc_backend:call(NodeName, ?MODULE, remote_rpc, [Myself, Mod, Fun, Args, Timeout, TraceId, SpanId], Timeout) of + {badrpc, RpcError} -> + ?set_status(error, <<"RPC_ERROR">>), + {error, {rpc, RpcError}}; + Error = {error, _} -> + ?set_status(error, <<"RESULT">>), + Error; + Result -> + ?set_status(ok), + Result + end; + Error -> + ?set_status(error, <<"NODE_NOT_FOUND">>), + Error + end + end). + +remote_rpc(Pid, Mod, Fun, Args, Timeout, TraceId, SpanId) -> + ParentCtx = otel_tracer:from_remote_span(TraceId, SpanId, 1), + otel_tracer:with_span(ParentCtx, opentelemetry:get_tracer(), ?FUN_NAME, #{}, + fun (Ctx) -> remote_rpc(Pid, Mod, Fun, Args, Timeout, Ctx) end). + +remote_rpc(Pid, Mod, Fun, Args, Timeout, Ctx) -> + ?set_attributes(#{remote_node => node(), module => Mod, function => Fun}), + try apply(Mod, Fun, Args) of + {badrpc, RpcError} -> + ?set_status(error, <<"RPC_ERROR">>), + {error, {rpc, RpcError}}; + Error = {error, _} -> + ?set_status(error, <<"RESULT">>), + Error; + Result -> + Result + catch + throw:Term:Stacktrace -> + ?set_status(error, <<"THROW">>), + {error, {throw, Term, Stacktrace}}; + exit:Reason:Stacktrace -> + ?set_status(error, <<"EXIT">>), + {error, {exit, Reason, Stacktrace}}; + error:Reason:Stacktrace -> + ?set_status(error, <<"RUNTIME">>), + {error, {runtime_error, Reason, Stacktrace}} + after + ?end_span() + end. + +get() -> + {ok, Node} = dreki_world:get_node(dreki_world:node()), + Node. + +urn() -> dreki_world:path(). + +uri() -> urn(). + +stores() -> dreki_world:stores(uri()). + +parents() -> + parents(urn()). + +parents(Node) -> + {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), + NodeDomain = dreki_world:path_to_domain(NodeUrn), + Vertices0 = dreki_world_dns:get_path({root, dreki_world:internal_domain()}, {node, NodeDomain}), + Vertices = lists:map(fun get_from_vertex/1, Vertices0), + [_Me | Parents] = lists:reverse(Vertices), + Parents. + +parent() -> + parent(urn()). + +parent(Node) -> + {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), + NodeDomain = dreki_world:path_to_domain(NodeUrn), + [Parent] = dreki_world_dns:in_neighbours({node, NodeDomain}), + get_from_vertex(Parent). + +neighbours() -> + neighbours(urn()). + +neighbours(Node) -> + {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), + NodeDomain = dreki_world:path_to_domain(NodeUrn), + [Parent] = dreki_world_dns:in_neighbours({node, NodeDomain}), + Neighbours = dreki_world_dns:out_neighbours(Parent), + lists:map(fun get_from_vertex/1, Neighbours -- [{node, NodeDomain}]). + +descendants() -> + descendants(urn()). + +descendants(Node) -> + {ok, #{node := NodeUrn}} = dreki_world:get_node(Node), + NodeDomain = dreki_world:path_to_domain(NodeUrn), + Descendants0 = dreki_world_dns:out_neighbours({node, NodeDomain}), + Descendants = case Descendants0 of + [{region, NodeDomain}] -> dreki_world_dns:out_neighbours({region, NodeDomain}); + D -> D + end, + lists:map(fun get_from_vertex/1, Descendants). + +get_from_vertex({root, Domain}) -> + {ok, Region} = dreki_world:get_region_from_dns_name(Domain), + Region; +get_from_vertex({region, Domain}) -> + {ok, Region} = dreki_world:get_region_from_dns_name(Domain), + Region; +get_from_vertex({node, Domain}) -> + {ok, Node} = dreki_world:get_node_from_dns_name(Domain), + Node. + +ensure_local_node() -> + case dreki_world:get_node(uri()) of + {ok, _} -> ok; + {error, {not_found, _}} -> create_local_node() + end. + +create_local_node() -> + dreki_world:create_node(uri(), #{}). diff --git a/apps/dreki/src/node/dreki_node_server.erl b/apps/dreki/src/node/dreki_node_server.erl new file mode 100644 index 0000000..c8bca51 --- /dev/null +++ b/apps/dreki/src/node/dreki_node_server.erl @@ -0,0 +1,21 @@ +-module(dreki_node_server). +-behaviour(partisan_gen_fsm). + +-export([start_link/1, send_event/2]). +-export([init/1]). +-export([wait/2]). + +-record(data, { }). + +start_link(Args) -> + partisan_gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). + +send_event(Name, Event) -> + partisan_gen_fsm:send_event(Name, Event). + +init(_) -> + {ok, wait, #data{}}. + +wait(Event, Data) -> + logger:info("node_server wait event: ~p", [Event]), + {next_state, wait, Data}. diff --git a/apps/dreki/src/node/rebar.conf b/apps/dreki/src/node/rebar.conf new file mode 100644 index 0000000..e69de29 diff --git a/apps/dreki/src/requests/dreki_requests.erl b/apps/dreki/src/requests/dreki_requests.erl new file mode 100644 index 0000000..d970dbd --- /dev/null +++ b/apps/dreki/src/requests/dreki_requests.erl @@ -0,0 +1,97 @@ +-module(dreki_requests). +-behaviour(dreki_store_namespace). +-export([start/0, version/0, valid_store/4, format_item/1, actions/0, actions/1, schemas/0, new/0, as_tuple/1, as_map/1]). +-export([record_name/0, record_attributes/0]). +-export([after_create/1]). +-export([new/1, create/2]). +-export([install/0]). + +-record(?MODULE, { + id, + version, + schema, + task_urn, + identity_urn, + node, + params + }). + +create(#{'@ns' := <<"tasks">>, '@location' := Loc, '@id' := TaskUrn}, Params0) -> + LocalStore = <>, + Params = Params0#{task_urn => TaskUrn}, + dreki_store:create(LocalStore, Params). + +install() -> + NodeUrn = dreki_node:urn(), + Urn = <>, + dreki_store:create_store(Urn, dreki_mnesia_store, #{}, #{}). + +record_name() -> + ?MODULE. + +record_attributes() -> + record_info(fields, ?MODULE). + +version() -> + 1. + +start() -> + ok. + +valid_store(_Ns, _Loc, _Name, _BackendMod) -> + ok. + +format_item(Item) -> + ok. + +actions() -> + []. + +actions(_) -> + []. + +schemas() -> + #{default => <<"request">>, + <<"request">> => #{ + default_version => <<"1.0">>, + <<"1.0">> => schemas(<<"request">>, <<"1.0">>) + } + }. + +schemas(<<"request">>, <<"1.0">>) -> + #{ + version => 'draft-06', + title => <<"Run Request">>, + type => object, + properties => #{ + id => #{type => string, <<"dreki:form">> => #{readonly => true}}, + node => #{type => string}, + params => #{type => object} + }, + required => [node, params] + }. + +new() -> + schemas(<<"request">>, <<"1.0">>). + +new(#{'@ns' := <<"tasks">>, '@id' := TaskUrn, id := TaskId, module := Module, params := TaskParams}) -> + Schema0 = schemas(<<"request">>, <<"1.0">>), + Schema = key_value:set([properties, params], TaskParams, Schema0), + {ok, Schema}. + +as_tuple(#{id := Id, task_urn := TaskUrn, node := Node, params := Params}) -> + #?MODULE{id = Id, version = undefined, schema = undefined, task_urn = TaskUrn, + node = Node, params = Params}. + +as_map(R = #?MODULE{}) -> + #{ + id => R#?MODULE.id, + version => R#?MODULE.version, + schema => R#?MODULE.schema, + task_urn => R#?MODULE.task_urn, + node => R#?MODULE.node, + params => R#?MODULE.params + }. + +after_create(Request) -> + ok. diff --git a/apps/dreki/src/runs/dreki_runs.erl b/apps/dreki/src/runs/dreki_runs.erl new file mode 100644 index 0000000..9509fa6 --- /dev/null +++ b/apps/dreki/src/runs/dreki_runs.erl @@ -0,0 +1 @@ +-module(dreki_runs). diff --git a/apps/dreki/src/storages/dreki_storages.erl b/apps/dreki/src/storages/dreki_storages.erl new file mode 100644 index 0000000..3a87347 --- /dev/null +++ b/apps/dreki/src/storages/dreki_storages.erl @@ -0,0 +1,98 @@ +-module(dreki_storages). +-behaviour(dreki_store_namespace). +-export([start/0, version/0, valid_store/4, format_item/1, actions/0, actions/1, schemas/0, as_tuple/1, as_map/1]). +-export([record_name/0, record_attributes/0]). +-export([after_create/1]). +-export([install_local_store/0, create_local_storage/4]). + +-record(?MODULE, { + id, + version, + schema, + name, + type, + handler, + params, + tags = [] + }). + +install_local_store() -> + NodeUrn = dreki_node:urn(), + Urn = <>, + dreki_store:create_store(Urn, dreki_mnesia_store, #{}, #{}). + +create_local_storage(Type, Handler, Params, Tags) -> + NodeUrn = dreki_node:urn(), + LocalStoreUrn = <>, + Storage = #{ + type => Type, + handler => Handler, + params => Params, + tags => Tags + }, + dreki_store:create(LocalStoreUrn, Storage). + +record_name() -> ?MODULE. +record_attributes() -> record_info(fields, ?MODULE). +version() -> 1. + +start() -> + ok. + +valid_store(_Ns, _Loc, _Name, _BackendMod) -> + ok. + +format_item(Item) -> + ok. + +actions() -> + []. + +actions(_) -> + []. + +schemas() -> + #{default => <<"storage">>, + <<"storage">> => #{ + default_version => <<"1.0">>, + <<"1.0">> => schemas(<<"storage">>, <<"1.0">>) + } + }. + +schemas(<<"storage">>, <<"1.0">>) -> + #{ + version => 'draft-06', + title => <<"Storage">>, + type => object, + required => [id, type, handler, params], + properties => #{ + id => #{type => string, <<"dreki:form">> => #{}}, + name => #{type => string}, + type => #{type => string, enum => [<<"fs">>, <<"os">>, <<"bs">>]}, + handler => #{type => string, enum => [<<"zfs">>, <<"fs">>]}, + params => #{type => object}, + tags => #{type => array, items => #{type => string}} + } + }. + +as_tuple(Map = #{id := Id, type := Type, handler := Handler, params := Params, tags := Tags}) -> + #?MODULE{id = Id, + name = maps:get(name, Map, undefined), + type = Type, + handler = Handler, + params = Params, + tags = maps:get(tags, Map, []) + }. + +as_map(R = #?MODULE{}) -> + #{ + id => R#?MODULE.id, + name => R#?MODULE.name, + type => R#?MODULE.type, + handler => R#?MODULE.handler, + params => R#?MODULE.params, + tags => R#?MODULE.tags + }. + +after_create(_Storage) -> + ok. diff --git a/apps/dreki/src/storages/dreki_storages_zfs.erl b/apps/dreki/src/storages/dreki_storages_zfs.erl new file mode 100644 index 0000000..fc8ce00 --- /dev/null +++ b/apps/dreki/src/storages/dreki_storages_zfs.erl @@ -0,0 +1,2 @@ +-module(dreki_storages_zfs). + diff --git a/apps/dreki/src/store/dreki_dets_store.erl b/apps/dreki/src/store/dreki_dets_store.erl new file mode 100644 index 0000000..aaa4c23 --- /dev/null +++ b/apps/dreki/src/store/dreki_dets_store.erl @@ -0,0 +1,73 @@ +-module(dreki_dets_store). + +-include("dreki.hrl"). + +-behaviour(dreki_store_backend). + +-export([start/0, start/5, checkout/1, checkin/1, stop/0, stop/1]). +-export([valid_store/5]). +-export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). + +-record(?MODULE, {tab, file}). + +start() -> ok. + +valid_store(_Namespace, _Location, _Name, _NSMod, _Args) -> ok. + +start(Namespace, _NsMod, Name, _XUrn, Args) -> + StoreName = {?MODULE, Namespace, Name}, + File = maps:get(file_name, Args, <>), + FileName = binary:bin_to_list(File), + case dets:open_file(StoreName, [{file, FileName}, {keypos, 2}]) of + {ok, Tab} -> {ok, #?MODULE{tab = Tab, file = FileName}}; + {error, Error} -> {error, {dets_open_failed, Error}} + end. + +checkout(#?MODULE{tab = Tab, file = FileName}) -> + case dets:open_file(Tab, [{file, FileName}, {keypos, 2}]) of + {ok, Tab} -> {ok, {dreki_dets_store_ref, Tab}}; + {error, Error} -> {error, {dets_open_failed, Error}} + end. + +checkin({dreki_dets_store_ref, Tab}) -> + dets:close(Tab). + +stop() -> ok. + +stop(#?MODULE{tab = Tab}) -> + dets:close(Tab). + +list({dreki_dets_store_ref, Tab}) -> + dets:foldl(fun (T, {ok, Ts}) -> {ok, [T | Ts]} end, {ok, []}, Tab). + +count({dreki_dets_store_ref, Tab}) -> + dets:foldl(fun (_T, {ok, Ct}) -> {ok, Ct + 1} end, {ok, 0}, Tab). + +exists({dreki_dets_store_ref, Tab}, Id) -> + dets:member(Tab, Id). + +get({dreki_dets_store_ref, Tab}, Id) -> + case dets:lookup(Tab, Id) of + [] -> {error, {task_not_found, Id}}; + [Task] -> {ok, Task} + end. + +delete({dreki_dets_store_ref, Tab}, Id) -> + dets:delete(Tab, Id). + +create({dreki_dets_store_ref, Tab}, Task = #dreki_task{persisted=false}) -> + case dets:insert_new(Tab, Task#dreki_task{persisted=true, dirty=false}) of + true -> {ok, Task#dreki_task{persisted=true, dirty=false}}; + false -> {error, {task_exists, Task#dreki_task.id}}; + {error, Error} -> {error, Error} + end. + +update(_Tab, Task = #dreki_task{persisted=false}) -> + {error, {task_not_created, Task#dreki_task.id}}; +update(_Tab, Task = #dreki_task{dirty=false}) -> + {ok, Task}; +update({dreki_dets_store_ref, Tab}, Task = #dreki_task{persisted=true, dirty=true}) -> + case dets:insert(Tab, Task#dreki_task{dirty=false}) of + ok -> {ok, Task#dreki_task{dirty=false}}; + {error, Error} -> {error, Error} + end. diff --git a/apps/dreki/src/store/dreki_mnesia_store.erl b/apps/dreki/src/store/dreki_mnesia_store.erl new file mode 100644 index 0000000..aceae09 --- /dev/null +++ b/apps/dreki/src/store/dreki_mnesia_store.erl @@ -0,0 +1,94 @@ +-module(dreki_mnesia_store). +-include("dreki.hrl"). +-include("dreki_otel.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("kernel/include/logger.hrl"). +-behaviour(dreki_store_backend). + +-export([start/0, start/5, checkout/1, checkin/1, stop/0, stop/1]). +-export([valid_store/5]). +-export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). + +start() -> + ok. + +valid_store(_NS, _Loc, _Name, _NSMod, _Args) -> + ok. + +start(Namespace, NsMod, Name, _XUrn, Args) -> + TabName = binary_to_atom(iolist_to_binary(["dreki_mnesia_store", "-", Namespace, "-", Name])), + case lists:member(TabName, mnesia:system_info(tables)) of + true -> + {ok, TabName}; + false -> + DefaultOptions = [{rocksdb_copies, [node()]}], + Options0 = maps:get(mnesia_table_options, Args, DefaultOptions), + Options = [{attributes, NsMod:record_attributes()}, {record_name, NsMod:record_name()} | Options0], + {atomic, ok} = mnesia:create_table(TabName, Options), + ?LOG_NOTICE(#{message => "Created mnesia table for store", mnesia_table => TabName, store_namespace => Namespace, store_name => Name}), + {ok, TabName} + end. + +checkout(TabName) -> + {ok, TabName}. + +checkin(_) -> + ok. + +stop() -> + ok. + +stop(_) -> + ok. + +list(TabName) -> + transaction(fun () -> + [get(TabName, Id) || Id <- iter_all(mnesia:first(TabName), TabName)] + end). + +count(TabName) -> + mnesia:table_info(TabName, size). + +get(TabName, Id) -> + transaction(fun() -> + case mnesia:read(TabName, Id) of + [Tuple] -> + Tuple; + [] -> + not_found + end + end). + +exists(TabName, Id) -> + case get(TabName, Id) of + not_found -> + false; + _ -> + true + end. + +create(Table, Tuple) -> + transaction(fun() -> + mnesia:write(Table, Tuple, write), + get(Table, element(2, Tuple)) + end). + +update(Table, Tuple) -> + create(Table, Tuple). + +delete(TabName, Id) -> + transaction(fun() -> + mnesia:delete(TabName, Id, write) + end). + +transaction(Fun) -> + ?with_span(?FUN_NAME, #{}, fun(_) -> {atomic, Result} = mnesia:transaction(Fun), Result end). + +iter_all(Id, TabName) -> + ?with_span(?FUN_NAME, #{}, fun(_) -> iter_all(Id, TabName, []) end). + +iter_all('$end_of_table', _, Acc) -> + Acc; +iter_all(Id, TabName, Acc) -> + [Tuple] = mnesia:read(TabName, Id), + iter_all(mnesia:next(TabName, Id), TabName, [Tuple | Acc]). diff --git a/apps/dreki/src/store/dreki_store.erl b/apps/dreki/src/store/dreki_store.erl new file mode 100644 index 0000000..63d87a8 --- /dev/null +++ b/apps/dreki/src/store/dreki_store.erl @@ -0,0 +1,475 @@ +-module(dreki_store). +-include("dreki.hrl"). +-include("dreki_plum.hrl"). +-include("dreki_otel.hrl"). +-define(BACKENDS_PT, {dreki_stores, backends}). +-compile({no_auto_import,[get/1]}). +-export([backends/0]). +-export([start/0]). +-export([namespaces/0, namespace/1]). +-export([stores/0, stores_local/0, stores/1, get_store/1, create_store/4, create_store/6]). +-export([list/1, get/1, new/1, create/2, update/2, delete/1]). +-export([store_as_map/1]). + +-behaviour(dreki_urn). +-export([expand_urn_resource_rest/4]). + +-export([callback/3, list_/2, get_/2, new_/2, create_/3, update_/3, delete_/2]). + +-record(store, {urn, xurn, namespace, name, namespace_mod, format, backend_mod, backend_params}). + +-type t() :: #store{}. +-type store() :: t() | dreki_uri() | dreki_expanded_uri(). + +backends() -> + [dreki_dets_store, dreki_world_store]. + +namespaces() -> + [ + {<<"tasks">>, dreki_tasks, #{}}, + {<<"requests">>, dreki_requests, #{}}, + {<<"storages">>, dreki_storages, #{}} + ]. + +namespace(Name) -> + lists:keyfind(Name, 1, namespaces()). + +start() -> + [ok = dreki_urn:register_namespace(NS, ?MODULE, Env) || {NS, _, Env} <- namespaces()], + [ok = BackendMod:start() || BackendMod <- backends()], + persistent_term:put(?BACKENDS_PT, #{}), + {Backends, Errors} = lists:foldr(fun (Store = #store{}, {Acc, Errs}) -> + case start_store_(Store) of + {ok, Backend} -> {maps:put(Store#store.urn, Backend, Acc), Errs}; + {error, Err} -> {Acc, [Err | Errs]} + end + end, {#{}, []}, stores_local()), + persistent_term:put(?BACKENDS_PT, Backends), + [logger:error("dreki_store: ~p", [Err]) || Err <- Errors], + ok. + +stores() -> + plum_db:fold(fun + ({_, Value}, Acc) -> [as_record(Value) | Acc]; + ({_, [Value]}, Acc) -> [as_record(Value) | Acc]; + ({_, ['$deleted']}, Acc) -> Acc + end, [], {?PLUM_DB_STORES_TAB, '_'}). + +stores_local() -> + lists:filter(fun is_local/1, stores()). + +stores(Namespace) -> + case namespace(Namespace) of + undefined -> {error, {namespace_not_found, Namespace}}; + {_, _, _} -> {ok, [Store || Store = #store{namespace = Namespace} <- stores()]} + end. + +start_store(Store = #store{}) -> + case start_store_(Store) of + {ok, Backend} -> + alarm_handler:clear_alarm({?MODULE, Store#store.urn}), + Pt = persistent_term:get(?BACKENDS_PT), + persistent_term:put(?BACKENDS_PT, maps:put(Store#store.urn, Backend, Pt)), + {ok, Backend}; + Error -> + alarm_handler:set_alarm({?MODULE, Store#store.urn}, {start_failed, Error}), + Error + end. + +start_store_(Store = #store{backend_mod = Mod}) -> + case is_local(Store) of + true -> start_store__(Store); + false -> {error, {store_not_local, Store#store.urn, dreki_node:urn()}} + end. +start_store__(Store) -> + case maps:get(Store#store.urn, persistent_term:get(?BACKENDS_PT), undefined) of + undefined -> start_store___(Store); + _Started -> {error, {store_already_started, Store#store.urn}} + end. +start_store___(Store = #store{backend_mod = Mod}) -> + case Mod:start(Store#store.namespace, Store#store.namespace_mod, Store#store.name, Store#store.urn, Store#store.backend_params) of + {ok, Backend} -> {ok, Backend}; + {error, Error} -> {error, {store_start_failed, Store#store.urn, Error}} + end. + +create_store(Urn, Module, ModuleParams, Params) when is_binary(Urn) -> + case dreki_urn:expand(Urn) of + Error = {error, _} -> Error; + {ok, XUrn} -> create_store(XUrn, Module, ModuleParams, Params) + end; +create_store(#{location := Location, resource := #{directory := #{namespace := NS, directory := Name}}}, Module, ModuleParams, Params) -> + create_store(NS, Location, Name, Module, ModuleParams, Params). + +create_store(Namespace, Location, Name, Module, ModuleParams, Params) -> + case lists:keyfind(Namespace, 1, namespaces()) of + undefined -> {error, {namespace_not_found, Namespace}}; + {_, NSMod, NSEnv} -> + case dreki_urn:expand(Location) of + Error = {error, _} -> Error; + {ok, #{location := Loc}} -> + Urn = <>, + case get_store(Urn) of + {ok, _} -> dreki_error:error(exists, [{urn, Urn}]); + Error = {error, _} -> Error; + not_found -> + case {NSMod:valid_store(Namespace, Location, Name, Module), Module:valid_store(Namespace, Location, Name, NSMod, ModuleParams)} of + {ok, ok} -> + {Prefix, Key} = {{?PLUM_DB_STORES_TAB, Loc}, {Namespace, Name}}, + %%ok = put_path(Urn, store, {Prefix, Key}), + ok = plum_db:put(Prefix, Key, #{urn => Urn, name => Name, namespace => Namespace, module => Module, module_params => ModuleParams, params => Params}), + get_store(Urn); + {{error, Error}, _} -> {error, {store_create_failed_namespace_rejected, {NSMod, Error}}}; + {_, {error, Error}} -> {error, {store_create_failed_store_rejected, {Module, Error}}} + end + end + end + end. + +expand_urn_resource_rest(Namespace, ResXUrn, Part, _Env) -> + logger:debug("Expanding resource ~p ~p", [Part, ResXUrn]), + expand_urn_resource_rest_(Namespace, ResXUrn, binary:split(Part, <<":">>, [global])). + +expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>]) -> + {ok, Res#{schemas => #{schemas => all}}}; +expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, Schema]) -> + {ok, Res#{schemas => #{schemas => Schema}}}; +expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, <<>>, <<>>]) -> + {ok, Res#{schema => #{schema => default}}}; +expand_urn_resource_rest_(Namespace, Res = #{namespace := _}, [<<"schemas">>, Schema, SchemaVer]) -> + {ok, Res#{schema => #{schema => Schema, version => SchemaVer}}}; +expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>]) -> + {ok, Res#{schemas => #{schemas => all}}}; +expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, Schema]) -> + {ok, Res#{schemas => #{schemas => Schema}}}; +expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, <<>>, <<>>]) -> + {ok, Res#{schema => #{schema => default}}}; +expand_urn_resource_rest_(Namespace, Res = #{directory := _}, [<<"schemas">>, Schema, SchemaVer]) -> + {ok, Res#{schema => #{schema => Schema, version => SchemaVer}}}; +expand_urn_resource_rest_(_, _, _) -> + error. + +-spec get_store(store()) -> {ok, t()} | {error, any()}. +get_store(Urn) when is_binary(Urn) -> + case dreki_urn:expand(Urn) of + {ok, Uri} -> get_store(Uri); + Error -> Error + end; +get_store(#{location := Location, resource := #{resource := #{namespace := NS, directory := Dir}}}) -> + get_store(Location, NS, Dir); +get_store(#{location := Location, resource := #{directory := #{namespace := NS, directory := Dir}}}) -> + get_store(Location, NS, Dir); +get_store(Fail) -> + {error, {uri_resolution_failed, Fail}}. + +get_store(Location, Namespace, Name) -> + case plum_db:get({?PLUM_DB_STORES_TAB, Location}, {Namespace, Name}) of + undefined -> not_found; + ['$deleted'] -> not_found; + [Val] -> {ok, as_record(Val)}; + Val -> {ok, as_record(Val)} + end. + +as_record([Map]) -> as_record(Map); +as_record(Store = #{urn := Urn, name := Name, namespace := Namespace, module := Module, module_params := ModuleParams}) -> + {ok, XUrn} = dreki_urn:expand(Urn), + {_, NSMod, _} = lists:keyfind(Namespace, 1, namespaces()), + Format = maps:get(format, Store, tuple), + #store{urn = Urn, xurn = XUrn, name = Name, namespace = Namespace, namespace_mod = NSMod, format = Format, backend_mod = Module, backend_params = ModuleParams}. + +store_as_map(#store{urn = Urn, name = Name, namespace = Namespace, namespace_mod = NSMod, backend_mod = Module, backend_params = ModuleParams}) -> + #{urn => Urn, name => Name, namespace => Namespace, namespace_mod => NSMod, backend_mod => Module, backend_params => ModuleParams}. + +-spec list(store()) -> {ok, dreki_store_namespace:collection()} | {error, any()}. +list(SArg) -> + get_store_(SArg, list_, []). + +new(SArg) -> + get_store_(SArg, new_, []). + +get(SArg) -> + get_store_(SArg, get_, []). + +create(SArg, Data) -> + get_store_(SArg, create_, [Data]). + +update(SArg, Data) -> + get_store_(SArg, update_, [Data]). + +delete(SArg) -> + get_store_(SArg, delete_, []). + +list_(#{resource := #{directory := _}}, Store) -> + handle_result(Store, collection, callback(Store, list, []), list); +list_(#{schema := _}, Store) -> + {todo, schema}; +list_(_, _) -> + not_supported. + +get_(#{resource := #{resource := #{id := Id}}}, Store) -> + handle_result(Store, single, callback(Store, get, [Id]), get); +get_(#{schema := _}, Store) -> + {todo, get_schema}; +get_(_, _) -> not_supported. + +new_(#{resource := #{directory := _}}, Store) -> + NSMod = Store#store.namespace_mod, + New = NSMod:new(), + {ok, New}. + +create_(XUrn = #{resource := #{directory := _}}, Store, Data0) -> + Id = maps:get(id, Data0, ksuid:gen_id()), + Data1 = Data0#{id => Id}, + case validate(XUrn, Data1) of + {ok, _} -> + Data = case Store#store.format of + tuple -> + Mod = Store#store.namespace_mod, + Mod:as_tuple(Data1); + map -> Data1 + end, + handle_result(Store, single, callback(Store, create, [Data]), create); + {error, Errors} -> + {error, #{code => validation_failed, status => 422, errors => jesse_error:to_json({error, Errors}, [])}} + end; +create_(_, _, _) -> + not_supported. + +update_(XUrn = #{resource := #{resource := _}}, Store, Data0) -> + case get(XUrn) of + {ok, Prev} -> + case validate(XUrn, Data0) of + {ok, _} -> + Data = case Store#store.format of + tuple -> + Mod = Store#store.namespace_mod, + Mod:as_tuple(Data0); + map -> Data0 + end, + handle_result(Store, single, callback(Store, update, [XUrn, Data]), update); + {error, Errors} -> + {error, #{code => validation_failed, status => 422, errors => jesse_error:to_json({error, Errors}, [])}} + end; + Error -> Error + end; +update_(_, _, _) -> + not_suppported. + +delete_(#{resource := _}, Store) -> + todo; +delete_(_, _) -> + not_supported. + +get_store_(StoreArg, Fun, Args) when is_binary(StoreArg) -> + logger:debug("Expanding in get_store_ :) ~p", [StoreArg]), + case dreki_urn:expand(StoreArg) of + Error = {error, _} -> Error; + {ok, XUrn} -> get_store_(XUrn, Fun, Args) + end; +get_store_(XUrn = #{resource := #{shemas := _}}, Fun, Args) -> + get_schema_(XUrn, Fun, Args); +get_store_(XUrn = #{resource := #{schema := _}}, Fun, Args) -> + get_schema_(XUrn, Fun, Args); +get_store_(XUrn = #{resource := _}, Fun, Args) -> + %%T = otel_tracer:start_span(opentelemetry:get_tracer(), <<"dreki_store">>, #{}), + %%?set_current_span(T), + ?with_span(opentelemetry:get_tracer(), #{}, fun(SpanCtx) -> + logger:debug("Getting store usually! ~p / Ctx: ~p / Span Ctx ~p / Cur Span Ctx ~p", [XUrn, otel_ctx:get_current(), SpanCtx, otel_tracer:current_span_ctx()]), + Result = case get_store(XUrn) of + Error = {error, _} -> Error; + {ok, Store} -> + case apply(?MODULE, Fun, [XUrn, Store | Args]) of + not_supported -> {error, {not_supported, Fun, maps:get(urn, XUrn)}}; + Out -> Out + end + end, + ?end_span(), + Result + end). + +is_local(#store{xurn = #{kind := region}}) -> true; +is_local(#store{xurn = #{kind := node, location := Location}}) -> Location =:= dreki_node:uri(). + +% TODO: Rescue execution so we always checkin +callback(Store = #store{}, Fun, Args) -> + callback(is_local(Store), Store, Fun, Args). + +% TODO: Rescue execution so we always checkin +callback(false, Store = #store{xurn = #{location := Location}}, Fun, Args) -> + logger:debug("dreki_store:callback: rpc:~p ~p ~p", [Location, Fun, Args]), + dreki_node:rpc(Location, ?MODULE, callback, [Store, Fun, Args]); +callback(true, #store{urn = Urn, backend_mod = Mod, backend_params = Params}, Fun, Args) -> + logger:debug("dreki_store:callback: local ~p ~p", [Fun, Args]), + case maps:get(Urn, persistent_term:get(?BACKENDS_PT), undefined) of + undefined -> dreki_error:error(store_backend_not_started, 503, <<"Store backend is not started">>); + Backend -> + {ok, B} = Mod:checkout(Backend), + Res = apply(Mod, Fun, [B | Args]), + ok = Mod:checkin(B), + Res + end. + +handle_result(_, _, {error, Err}, _) -> {error, Err}; +handle_result(Store, collection, {ok, Collection}, PostCallback) when is_list(Collection) -> + handle_result(Store, collection, Collection, PostCallback); +handle_result(Store, collection, Collection, PostCallback) when is_list(Collection) -> + {ok, handle_collection_result(Collection, Store, [], PostCallback)}; +handle_result(Store, single, Item, PostCallback) -> + handle_single_result(Item, Store, PostCallback); +handle_result(Store, single, {ok, Item}, PostCallback)-> + handle_single_result(Item, Store, PostCallback). + +handle_collection_result([Item | Rest], Store, Acc0, PostCallback) -> + Acc = case handle_single_result(Item, Store, PostCallback) of + ignore -> Acc0; + {ok, I} -> [I | Acc0] + end, + handle_collection_result(Rest, Store, Acc, PostCallback); +handle_collection_result([], Store, Acc, _PostCallback) -> + format_collection(Acc, Store). + +handle_single_result(Tuple, Store = #store{format = tuple, namespace_mod = Mod}, PostCallback) when is_tuple(Tuple) -> + handle_single_result(Mod:as_map(Tuple), Store, PostCallback); +handle_single_result(Item = #{id := LId}, Store = #store{namespace_mod = Mod}, PostCallback) -> + case Mod:format_item(Item) of + ok -> format_item(Item, Item, Store, PostCallback); + {ok, M} -> format_item(M, Item, Store, PostCallback); + Err -> Err + end. + +format_item(Item = #{id := LId}, OriginalItem, Store = #store{urn = Urn, namespace = NS, namespace_mod = Mod}, PostCallback) -> + {ok, XUrn} = dreki_urn:expand(Urn), + SelfUrn = <>, + AtLinks = #{ + self => SelfUrn, + parent => Urn + }, + AllAtLinks = maps:merge(AtLinks, maps:get('@links', Item, #{})), + Actions = [#{id => Id, title => Title, new => New, create => Create} || {Id, Title, New, Create} <- Mod:actions(OriginalItem)], + I = #{'@id' => SelfUrn, '@ns' => NS, '@location' => maps:get(location, XUrn), '@links' => AllAtLinks, '@actions' => Actions}, + FullItem = maps:merge(I, Item), + case PostCallback of + create -> Mod:after_create(FullItem); + _ -> undefined + end, + {ok, FullItem}. + +format_collection(Data, Store = #store{urn = Urn}) -> + #{'@links' => #{self => Urn}, + data => Data}. + +get_schema_(XUrn = #{resource := #{directory := #{directory := _, namespace := NS}, schemas := Schemas}}, Fun, Args) -> + get_schema_(XUrn#{resource => #{namespace => NS, schemas => Schemas}}, Fun, Args); +get_schema_(XUrn = #{resource := #{directory := #{directory := _, namespace := NS}, schema := Schema}}, Fun, Args) -> + get_schema_(XUrn#{resource => #{namespace => NS, schema => Schema}}, Fun, Args); +get_schema_(#{urn := Urn, resource := #{namespace := NS, schemas := #{schemas := all}}}, list_, _) -> + {_, NSMod, _} = namespace(NS), + Schemas = maps:fold(fun + (default, Value, Acc) -> maps:put(default, Value, Acc); + (SchemaName, Content, Acc) -> + SUrn = <>, +logger:debug("Content is ~p", [Content]), + Schema = maps:fold(fun + (default_version, Value, CAcc) -> maps:put(default_version, Value, CAcc); + (Vsn, _, CAcc) when is_binary(Vsn) -> + Version = #{id => <>, version => Vsn}, + maps:put(versions, [Version | maps:get(versions, CAcc, [])], CAcc) + end, #{id => SUrn}, Content), + maps:put(schemas, [Schema | maps:get(schemas, Acc, [])], Acc) + end, #{}, NSMod:schemas()), + {ok, Schemas}; +get_schema_(#{urn := Urn, resource := #{namespace := NS, schemas := #{schemas := Query}}}, list_, _) -> + logger:debug("Querying Schemas ~p", [Query]), + {_, NSMod, _} = namespace(NS), + case maps:get(Query, NSMod:schemas(), not_found) of + not_found -> {error, not_found}; + SchemaSpec -> + Schema = maps:fold(fun + (default, Value, CAcc) -> maps:put(default_version, Value, CAcc); + (Vsn, _, CAcc) -> + Version = #{id => <>, version => Vsn}, + maps:put(versions, [Version | maps:get(versions, CAcc, [])], CAcc) + end, #{id => Urn}, SchemaSpec), + {ok, Schema} + end; +get_schema_(#{urn := Urn, resource := #{namespace := NS, schema := #{schema := default}}}, get_, Args) -> + logger:debug("Looking for default schema!"), + {_, NSMod, _} = namespace(NS), + Schemas = NSMod:schemas(), + SchemaName = maps:get(default, Schemas, not_found), + logger:debug("=+> SchemaName ~p IN ~p", [SchemaName, Schemas]), + case maps:get(SchemaName, Schemas, not_found) of + not_found -> not_found; + Schema -> + case maps:get(default_version, Schema, not_found) of + not_found -> not_found; + SchemaVer -> + NewUrn0 = binary:replace(Urn, <<"::schemas::">>, <<>>), + NewUrn = <>, + {ok, XUrn} = dreki_urn:expand(NewUrn), + logger:debug("Looking up default schema as ~p (~p)", [NewUrn, XUrn]), + get_schema_(XUrn, get_, Args) + end + end; +get_schema_(XUrn = #{resource := #{namespace := NS, schema := #{schema := SchemaName, version := Version}}}, get_, _) -> + {_, NSMod, _} = namespace(NS), + case maps:get(SchemaName, NSMod:schemas(), not_found) of + not_found -> not_found; + Schema -> format_schema(maps:get(Version, Schema, not_found), SchemaName, Version, XUrn) + end; +get_schema_(XUrn = #{urn := Urn}, Method, _) -> + {error, {unsupported, Urn, XUrn, Method}}. + +format_schema(not_found, _, _, _) -> + {error, not_found}; +format_schema(Schema, SchemaName, SchemaVersion, XUrn = #{urn := Urn}) -> + {ok, maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, SchemaName, SchemaVersion, XUrn), maps:put(K2, V2, Acc) end, #{}, Schema#{<<"$id">> => Urn})}. + +format_schema_field('$$ref', SubPath, PSN, PSV, #{urn := Urn}) -> + [SN, SV] = binary:split(SubPath, <<":">>), + NewUrn = binary:replace(Urn, <>, <>), + {'$ref', NewUrn}; +format_schema_field(Key, Map, PSn, PSv, XUrn) when is_map(Map) -> + {Key, maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, PSn, PSv, XUrn), maps:put(K2, V2, Acc) end, #{}, Map)}; +format_schema_field(Key, List, PSn, PSv, XUrn) when is_list(List) -> + {Key, lists:map(fun (V) -> format_schema_field(V, PSn, PSv, XUrn) end, List)}; +format_schema_field(K, V, _, _, _) -> + {K, V}. +format_schema_field(Map, PSn, PSv, XUrn) when is_map(Map) -> + maps:fold(fun (K, V, Acc) -> {K2, V2} = format_schema_field(K, V, PSn, PSv, XUrn), maps:put(K2, V2, Acc) end, #{}, Map); +format_schema_field(List, PSn, PSv, XUrn) when is_list(List) -> + lists:map(fun (V) -> {_, Va} = format_schema_field(V, PSn, PSv, XUrn), Va end, List); +format_schema_field(Value, _, _, _) -> + Value. + +schema_to_urn(NameAndVer, XUrn) -> + [Name, Ver] = binary:split(NameAndVer, <<":">>), + Namespace = case XUrn of + #{resource := #{namespace := NS}} -> NS; + #{resource := #{directory := #{namespace := NS}}} -> NS; + #{resource := #{resource := #{namespace := NS}}} -> NS + end, + dreki_urn:to_urn(XUrn#{resource => #{namespace => NS, schema => #{schema => Name, version => Ver}}}). + +validate(XUrn, Data) -> + Res = case maps:get(<<"@schema">>, Data, undefined) of + undefined -> get(XUrn#{resource => #{namespace => get_xurn_namespace(XUrn), schema => #{schema => default}}}); + Urn = <<"dreki:", _/binary>> -> get(Urn); + SchemaNameAndVer -> get(schema_to_urn(SchemaNameAndVer, XUrn)) + end, + case Res of + {ok, Schema} -> validate_(XUrn, Schema, Data); + Error -> Error + end. + +get_xurn_namespace(#{resource := #{namespace := NS}}) -> NS; +get_xurn_namespace(#{resource := #{directory := #{namespace := NS}}}) -> NS; +get_xurn_namespace(#{resource := #{resource := #{namespace := NS}}}) -> NS. + +schema_loader(SchemaRef) -> + logger:debug("Trying to load schema ~p", [SchemaRef]), + get(SchemaRef). + +validate_(XUrn, Schema, Data) -> + JesseOpts = [{allowed_errors, infinity}, + {schema_loader_fun, fun schema_loader/1}], + jesse:validate_with_schema(Schema, Data, JesseOpts). diff --git a/apps/dreki/src/store/dreki_store_backend.erl b/apps/dreki/src/store/dreki_store_backend.erl new file mode 100644 index 0000000..55db90e --- /dev/null +++ b/apps/dreki/src/store/dreki_store_backend.erl @@ -0,0 +1,33 @@ +-module(dreki_store_backend). +-include("dreki.hrl"). + +-type t() :: module(). + +-type backend_ref() :: any(). +-type backend_checkout_ref() :: any(). + +-type args() :: any(). + +-callback valid_store(dreki_expanded_uri(), Namespace_mod :: module(), Args :: any()) -> ok | {error, any()}. + +-callback start() -> ok | {error, ba}. + +-callback start(binary(), binary(), dreki_urn:urn(), args()) -> {ok, backend_ref()} | {error, any()}. + +-callback stop() -> ok. + +-callback stop(backend_ref()) -> ok. + +-callback checkout(backend_ref()) -> {ok, backend_checkout_ref()} | {error, any()}. + +-callback checkin(backend_checkout_ref()) -> ok. + +-callback list(backend_checkout_ref()) -> {ok, dreki_store_namespace:collection()}. + +-callback get(backend_checkout_ref(), dreki_id()) -> {ok, dreki_store_namespace:item()} | not_found | {error, any()}. + +-callback create(backend_checkout_ref(), dreki_store_namespace:item()) -> ok | {error, any()}. + +-callback update(backend_checkout_ref(), dreki_store_namespace:item()) -> ok | not_found | {error, any()}. + +-callback delete(backend_checkout_ref(), dreki_id()) -> ok | not_found | {error, any()}. diff --git a/apps/dreki/src/store/dreki_store_namespace.erl b/apps/dreki/src/store/dreki_store_namespace.erl new file mode 100644 index 0000000..3095ef7 --- /dev/null +++ b/apps/dreki/src/store/dreki_store_namespace.erl @@ -0,0 +1,14 @@ +-module(dreki_store_namespace). +-include("dreki.hrl"). + +-type t() :: module(). + +-type item() :: any(). +-type collection() :: [item()]. +-type name() :: binary(). + +-callback start() -> ok | {error, any()}. +-callback format_item(item()) -> ok | {ok, item()} | ignore. +-callback valid_store(name(), Location :: dreki_urn:urn(), StoreName :: binary(), BackendModule :: module()) -> ok | {error, any()}. +-callback version() -> non_neg_integer(). +-callback schemas() -> #{default := Id :: binary(), Id :: binary() => #{default_version := Vsn :: binary(), Vsn :: binary => Schema :: #{}}}. diff --git a/apps/dreki/src/tasks/dreki_task.erl b/apps/dreki/src/tasks/dreki_task.erl new file mode 100644 index 0000000..b762aea --- /dev/null +++ b/apps/dreki/src/tasks/dreki_task.erl @@ -0,0 +1,58 @@ +-module(dreki_task). + +-include("dreki.hrl"). + +-type new_params() :: #{handler := dreki_task_handler(), + id => dreki_id(), + description => binary(), + params => #{}}. + +-export([new/1, validate/1, to_map/1]). +-export([id/1, description/1, handler/1, params/1]). +-export([description/2, handler/2, params/2]). + +-spec new(new_params()) -> {ok, dreki_task()} | {error, Reason::term()}. +new(Map) -> + Id = maps:get(id, Map, dreki_id:get()), + Description = maps:get(description, Map, undefined), + Params = maps:get(params, Map, #{}), + Handler = maps:get(handler, Map, undefined), + Task = #dreki_task{id=Id, handler=Handler, params=Params, description=Description, + persisted=false, dirty=true}, + validate(Task). + +-spec validate(dreki_task()) -> {ok, dreki_task()} | {error, Reason::term()}. +validate(#dreki_task{handler = undefined}) -> + {error, {required, handler}}; +validate(Task = #dreki_task{id = Id, handler = _Handler}) -> + case dreki_id:valid(Id) of + {error, Err} -> {error, Err}; + ok -> {ok, Task} + end. + +id(Task = #dreki_task{}) -> + Task#dreki_task.id. + +description(Task = #dreki_task{}) -> + Task#dreki_task.description. + +handler(Task = #dreki_task{}) -> + Task#dreki_task.handler. + +params(Task = #dreki_task{}) -> + Task#dreki_task.params. + +handler(Task = #dreki_task{}, NewHandler) -> + Task#dreki_task{handler=NewHandler, dirty=true}. + +description(Task = #dreki_task{}, NewDescription) -> + Task#dreki_task{description=NewDescription, dirty=true}. + +params(Task = #dreki_task{}, NewParams) -> + Task#dreki_task{params=NewParams, dirty=true}. + +to_map(Task = #dreki_task{id = Id, handler = Handler, description = Description, params = Params}) -> + #{id => Id, + handler => Handler, + description => Description, + params => Params}. diff --git a/apps/dreki/src/tasks/dreki_tasks.erl b/apps/dreki/src/tasks/dreki_tasks.erl new file mode 100644 index 0000000..750aed0 --- /dev/null +++ b/apps/dreki/src/tasks/dreki_tasks.erl @@ -0,0 +1,166 @@ +-module(dreki_tasks). +-include("dreki.hrl"). + +-behaviour(dreki_store_namespace). +-export([start/0, version/0, valid_store/4, format_item/1, actions/0, actions/1, schemas/0, new/0, as_tuple/1, as_map/1]). + +%% old stuff +-export([resolve/1, exists/1, read_uri/2]). + +start() -> ok. + +valid_store(_Namespace, _Location, _Name, _BackendMod) -> ok. + +format_item(Item) -> ok. + +handlers() -> [dreki_tasks_script, dreki_tasks_cloyster, drekid_function]. + +-record(t, { + id, + version, + schema, + module, + params +}). + +version() -> 1. + +new() -> + #{ + <<"@schema">> => <<"task:1.0">>, + <<"id">> => ksuid:gen_id(), + <<"module">> => <<"dreki_tasks_cloyster">>, + <<"params">> => #{ + <<"@schema">> => <<"cloyster-task:1.0">>, + <<"script">> => <<>> + } + }. + +actions() -> + [ + {new, <<"New task">>, {dreki_tasks, new, []}, {dreki_store, create, []}} + ]. + +actions(_) -> + [ + {request, <<"Request run">>, + {dreki_requests, new, []}, + {dreki_store, create, []}} + ]. + +schemas() -> + Subs = lists:foldr(fun (Handler, Acc) -> maps:merge(Acc, Handler:schemas()) end, + #{}, handlers()), + maps:merge(Subs, #{ + default => <<"task">>, + <<"task">> => schemas(task) + }). + +schemas(task) -> + #{ + default_version => <<"1.0">>, + <<"1.0">> => schemas(task, <<"1.0">>) + }. + +schemas(task, <<"1.0">>) -> + Handlers = handlers(), + Manifests0 = lists:map(fun (Handler) -> {Handler, Handler:schema_field(handler_manifest)} end, Handlers), + Manifests = lists:foldr(fun + ({Handler, undefined}, Acc) -> Acc; + ({Handler, Schema}, Acc) -> + Schemas = maps:fold(fun + (Atom, _, Acc) when is_atom(Atom) -> Acc; + (Vsn, _, Acc) -> [#{'$$ref' => <>} | Acc] + end, [], maps:get(Schema, Handler:schemas())), + Acc ++ Schemas + end, [], Manifests0), + + #{ + version => 'draft-06', + title => <<"Task">>, + type => object, + properties => #{ + id => #{type => string, <<"dreki:form">> => #{default => {ksuid, gen_id, []}}}, + schema => #{type => string}, + name => #{type => string, title => <<"Name">>}, + handler => #{type => string, title => <<"Handler">>, enum => handlers()}, + handler_manifest => #{'$ref' => <<"handler_manifest">>} + }, + '$defs' => #{ + <<"handler_manifest">> => #{anyOf => Manifests} + }, + required => [handler, handler_manifest] + }. + +as_tuple(#{id := Id, module := Module, params := Params}) -> + #t{id = Id, version = undefined, schema = undefined, + module = Module, params = Params}. + +as_map(Task = #t{}) -> + #{ + id => Task#t.id, + version => Task#t.version, + schema => Task#t.schema, + module => Task#t.module, + params => Task#t.params + }. + +%% old stuff + +read_uri(undefined, Uri) -> + {ok, #{stores => #{}}}; +read_uri(Path, Uri) -> + case binary:split(Path, <<":">>, [global]) of + [<<>>, <<>>] -> {error, invalid_resource}; + [<<>>, Id] -> {ok, #{resolve => #{kind => tasks, id => Id}}}; + [S] -> {ok, #{store => #{kind => tasks, id => S}}}; + [S, <<>>] -> {ok, #{store => #{kind => tasks, id => S}}}; + [S, Id] -> {ok, #{resource => #{kind => tasks, store => S, id => Id}}}; + R -> {error, {invalid_address, {Path, R}}} + end. + +all(Uri) when is_binary(Uri) -> + all(dreki_world:read_uri(Uri)); +all({ok, Uri = #{kind := tasks, uri := Uri, resource := Res = #{store := #{id := _S}}}}) -> + {ok, Mod, Store} = dreki_node:get_store(Uri), + Mod:all(Store); +all({ok, Uri}) -> {error, unresolvable_uri, Uri}; +all(Error = {error, _}) -> Error. + + +resolve(Id) -> + Path = dreki_world:path(), + case binary:match(Id, <>) of + {0,End} -> + StoreAndId = binary:part(Id, {End, byte_size(Id) - End}), + [StoreN, LId] = binary:split(StoreAndId, <<":">>), + {ok, {node(), StoreN, LId}} + end. + +exists({Node, StoreN, LId}) when Node =:= node() -> + #{mod := Mod, args := Args} = maps:get(StoreN, load_local_stores()), + {ok, Db} = Mod:open(Args), + Mod:exists(Db, LId); +exists(N = {Node, _, _}) -> + erpc:call(Node, dreki_tasks, exists, [N]); +exists(Id) when is_binary(Id) -> + case resolve(Id) of + {ok, N} -> exists(N); + Error = {error, _} -> Error + end. + +load_local_stores() -> + {ok, Val} = application:get_env(dreki, local_tasks_stores), + _World = #{path := Path, me := Me} = dreki_world:to_map(), + MapFn = fun ({Name, Mod, Args, Env}, Acc) -> + Store = #{local => true, + node => Me, + path => <>, + mod => Mod, + args => Args, + env => Env, + name => Name + }, + maps:put(Name, Store, Acc) + end, + lists:foldr(MapFn, #{}, Val). diff --git a/apps/dreki/src/tasks/dreki_tasks_cloyster.erl b/apps/dreki/src/tasks/dreki_tasks_cloyster.erl new file mode 100644 index 0000000..3fb045d --- /dev/null +++ b/apps/dreki/src/tasks/dreki_tasks_cloyster.erl @@ -0,0 +1,21 @@ +-module(dreki_tasks_cloyster). +-export([schemas/0, schema_field/1]). + +schema_field(handler_manifest) -> <<"cloyster-task">>. + +schemas() -> + #{ + <<"cloyster-task">> => #{ + default_version => <<"1.0">>, + <<"1.0">> => #{ + version => 'draft-06', + title => <<"Cloyster Task Definition">>, + type => object, + properties => #{ + <<"script">> => #{type => string} + }, + required => [script] + } + } + }. + diff --git a/apps/dreki/src/tasks/dreki_tasks_script.erl b/apps/dreki/src/tasks/dreki_tasks_script.erl new file mode 100644 index 0000000..8eeb563 --- /dev/null +++ b/apps/dreki/src/tasks/dreki_tasks_script.erl @@ -0,0 +1,24 @@ +-module(dreki_tasks_script). +-export([schemas/0, schema_field/1]). + +schema_field(handler_manifest) -> <<"script-task">>. + +schemas() -> + #{ + <<"script-task">> => #{ + default_version => <<"1.0">>, + <<"1.0">> => #{ + version => 'draft-06', + title => <<"Executable Script Task">>, + type => object, + properties => #{ + <<"id">> => #{type => string, <<"dreki:form">> => #{default => generate_id}}, + <<"name">> => #{type => string}, + <<"description">> => #{type => string, <<"dreki:form">> => #{input => textarea, textarea_mode => markdown}}, + <<"executable">> => #{type => string, default => <<"/bin/sh">>}, + <<"script">> => #{type => string, <<"dreki:form">> => #{input => textarea}} + }, + required => [script] + } + } + }. diff --git a/apps/dreki/src/world/dreki_world.erl b/apps/dreki/src/world/dreki_world.erl new file mode 100644 index 0000000..437b6c8 --- /dev/null +++ b/apps/dreki/src/world/dreki_world.erl @@ -0,0 +1,366 @@ +-module(dreki_world). + +-export([ensure_consistency/0, index/2]). +-export([get_path/1, get_path/2, paths/0, put_path/3]). +-export([get_region_from_dns_name/1, get_region/1, get_region/2, get_region/3, create_region/1, put_region/3]). +-export([node/0, nodes/0, get_node_from_dns_name/1, get_node/1, create_node_from_dns_name/2, create_node/2, put_node/3]). +-export([refresh_index/2, get/2, get/3, maybe_put_index/5]). +-export([namespace_to_module/1]). +-export([path_to_domain/1]). +-export([to_map/0, root_domain/0, internal_domain/0, domain/0, hierarchy/0, parent/0, me/0, path/0, root_path/0, strip_root_path/1, put_root_path/1, read_uri/1, domain_to_path/1]). + +ensure_consistency() -> + Dns = dreki_world_dns:as_map(), + Vertices = maps:get(vertices, Dns), + ensure_consistency(Vertices, []). + +ensure_consistency([#{type := root, name := Root} | Rest], Acc) -> + Acc = case get_region_from_dns_name(Root) of + {error, {not_found, _}} -> + ok = create_region_from_dns_name(Root), + put_region(Root, root, true), + [{root, Root} | Acc]; + _ -> + Acc + end, + Res = [Log || Kind <- [tasks], Log = {create, _} <- [ensure_global_root_stores(Root, Kind)]], + ensure_consistency(Rest, Res ++ Acc); + +ensure_consistency([#{type := region, name := Region} | Rest], Acc) -> + case get_region_from_dns_name(Region) of + {error, {not_found, _}} -> + ensure_consistency(Rest, [{Region, create_region_from_dns_name(Region)} | Acc]); + _ -> + ensure_consistency(Rest, Acc) + end; +ensure_consistency([_ | Rest], Acc) -> + ensure_consistency(Rest, Acc); +ensure_consistency([], Acc) -> + Acc. + +ensure_global_root_stores(Root, Kind) -> + {ok, Region} = get_region_from_dns_name(Root), + RegionUri = maps:get(uri, Region), + KindB = atom_to_binary(Kind), + StoreUri = <>, + case dreki_store:get_store(StoreUri) of + {ok, _store} -> ok; + _ -> {create, {Root,Kind,dreki_store:create_store(StoreUri, dreki_world_store, #{}, #{})}} + end. + +put_index(IdxKey, IdxValue, Uri, Data) -> + plum_db:put({IdxKey, IdxValue}, Uri, Data). + +maybe_put_index(Kind, IdxKey, IdxValue, Uri, Data) -> + case lists:member(IdxKey, index_fields(Kind)) of + true -> put_index(index_key(IdxKey), IdxValue, Uri, Data); + false -> ok + end. + +index_key(roles) -> 'idx:roles'; +index_key(tags) -> 'idx:tags'. + +index(Key, Value) -> + Idx = index_key(Key), + plum_db:fold(fun + ({_, ['$deleted']}, Acc) -> Acc; + ({Key, Val}, Acc) -> + [{Key, Val} | Acc] + end, [], {Idx, Value}). + +index_fields(node) -> [roles, tags]; +index_fields(region) -> [roles, tags]; +index_fields(_) -> [roles, tags]. + +refresh_index(Kind, Map) when is_atom(Kind) -> + refresh_index(index_fields(Kind), Kind, Map). +refresh_index([Field | Rest], Kind, Map = #{uri := URI}) -> + Values = case maps:get(Field, Map, undefined) of + undefined -> []; + V when is_list(V) -> V; + V -> [V] + end, + [maybe_put_index(Kind, Field, V, URI, undefined) || V <- Values], + refresh_index(Rest, Kind, Map); +refresh_index([], _, _) -> + ok. + +clean_path(Path) -> + Len = byte_size(Path) - 1, + case Path of + <> -> P; + P -> P + end. + +put_path(Path0, Kind, Uri) -> + Path = clean_path(Path0), + {ok, Links} = get({paths, Path}, Path, [{default, #{}}]), + case maps:find(Kind, Links) of + {ok, _} -> {error, {exists, {Path, Kind}}}; + error -> + ok = plum_db:put({paths, Path}, Path, Links#{Kind => Uri}), + ok + end. + +get_path(Path0) -> + Path = clean_path(Path0), + case get({paths, Path}, Path) of + not_found -> {error, {not_found, {path, Path}}}; + {ok, Data} -> {ok, Data} + end. + +get_path(Path, Kind) -> + case get_path(Path) of + {ok, Map} -> + case maps:get(Kind, Map, undefined) of + undefined -> {error, {not_found, {Path, Kind}}}; + Value -> {ok, Value} + end; + Error = {error, _} -> Error + end. + +paths() -> + plum_db:fold(fun + ({_, ['$deleted']}, Acc) -> Acc; + ({Path, Value}, Acc) -> + maps:put(Path, Value, Acc) + end, #{}, {paths, '_'}). + +get(Prefix, Key) -> + get(Prefix, Key, []). + +get(Prefix, Key, Opts) -> + case plum_db:get(Prefix, Key, Opts) of + undefined -> not_found; + ['$deleted'] -> not_found; + Val -> {ok, Val} + end. + +region_uri_from_dns_name(Name) -> + Root = internal_domain(), + if + Root =:= Name -> root_path(); + true -> domain_to_path(Name) + end. + +get_region_from_dns_name(Name) -> + get_region(region_uri_from_dns_name(Name)). + +get_region(Path) -> + case get_path(Path, region) of + {ok, _} -> + Region = plum_db:fold(fun + ({_, ['$deleted']}, M) -> M; + ({{_, K}, V}, M) -> maps:put(K, V, M) + end, #{uri => Path}, {regions, Path}), + {ok, Region}; + Error -> Error + end. + +get_region(Path, Key) -> + get_region(Path, Key, undefined). + +get_region(Path, Key, GetOpts) -> + case get_path(Path, region) of + {ok, _} -> get({regions, Path}, {Path, Key}, GetOpts); + Error -> Error + end. + +create_region_from_dns_name(Name) -> + create_region(region_uri_from_dns_name(Name)). + +create_region(Path) -> + case get_region(Path) of + {ok, _} -> {error, {exists, {region, Path}}}; + {error, {not_found, _}} -> + ok = put_path(Path, region, Path), + ok = put_region(Path, region, Path), + {ok, R} = get_region(Path), + ok = refresh_index(region, R), + ok + end. + +put_region(Path, Key, Value) -> + case get_path(Path, region) of + {ok, _} -> + ok = plum_db:put({regions, Path}, {Path, Key}, Value), + ok = maybe_put_index(region, Key, Value, Path, undefined); + Error -> Error + end. + +nodes() -> + lists:foldr(fun (Domain, Acc) -> + case get_node_from_dns_name(Domain) of + {ok, Node} -> [Node | Acc]; + _ -> Acc + end + end, [], dreki_world_dns:nodes()). + +node() -> + dreki_world:domain_to_path(dreki_world:domain()). + +get_node_from_dns_name(Name) -> + get_node(domain_to_path(Name)). + +get_node(Path) -> + case get_path(Path, node) of + {ok, _} -> + Node = plum_db:fold(fun + ({_, ['$deleted']}, M) -> M; + ({{_, K}, [V]}, M) -> maps:put(K, V, M) + end, #{uri => Path}, {nodes, Path}), + {ok, Node}; + Error -> Error + end. + +create_node_from_dns_name(Name, Params) -> + create_node(domain_to_path(Name), Params). + +create_node(Path, Params) -> + case get_path(Path, node) of + {ok, _} -> {error, {exists, {node, Path}}}; + {error, {not_found, _}} -> + ok = put_path(Path, node, Path), + ok = put_node(Path, node, Path), + [ok = put_node(Path, Key, Value) || {Key, Value} <- maps:to_list(Params)], + ok + end. + +put_node(Path, Key, Value) -> + case get_path(Path, node) of + {ok, _} -> plum_db:put({nodes, Path}, {Path, Key}, Value); + Error -> Error + end. + +domain_to_path(Domain) -> + Clean = strip_domain(Domain, internal_domain()), + Parts = binary:split(Clean, <<".">>, [global]), + Components = [<<"dreki">>, root_domain(), internal_subdomain()] ++ lists:reverse(Parts), + join(<<":">>, Components). + +path_to_domain(<<"dreki:", Rest/binary>>) -> + [Location | _] = binary:split(Rest, <<"::">>), + Components = binary:split(Location, <<":">>, [global]), + join(<<".">>, lists:reverse(Components)). + +root_domain() -> + get_env(root_domain). + +internal_domain() -> + get_env(internal_domain). + +domain() -> + get_env(domain). + +read_uri(U = <<"dreki:", _/binary>>) -> + read_uri(U, root_path()); +read_uri(Invalid) -> + {error, {bad_type, Invalid}}. +%% Remove root +%%read_uri(U, Root) -> +%% read_uri(U, Root, Rest). +%%read_uri(U, Root) -> +%% {error, #{message => <<"URI outside of domain">>, uri => U, domain => Root}}. +%% Extract hierarchy +read_uri(U, Root) -> + {Hier, Res} = case binary:split(U, <<"::">>) of + [H, R] -> {H, R}; + [H] -> {H, undefined} + end, + case get_path(H) of + {ok, Thing} -> read_uri_(U, Root, H, Res, Thing); + Error -> Error + end. + +read_uri_(U, Root, Path, ResPath, Thing) -> + Kind = case maps:keys(Thing) of + [K] -> K; + Ks -> Ks + end, + RegionStoreHint = case {maps:get(region, Thing, false), maps:get(node, Thing, false)} of + {false, _} -> global; + {_, false} -> global; + {_, _} -> local + end, + StoreHints = lists:foldr(fun + (node, Acc) -> maps:put(node, local, Acc); + (region, Acc) -> maps:put(region, RegionStoreHint, Acc) + end, #{}, maps:keys(Thing)), + Uri = #{domain => Root, path => Path, kind => Kind, store_hints => StoreHints, uri => U}, + case read_resource_uri(ResPath, Uri) of + {ok, Resource} -> + {ok, Uri#{resource => Resource}}; + {error, invalid_resource} -> + {error, #{message => <<"Invalid resource">>, uri => U, resource => ResPath}}; + {error, invalid_namespace, NS} -> + {error, #{message => <<"Invalid namespace">>, uri => U, namespace => NS}} + end. + +read_resource_uri(undefined, _) -> + {ok, undefined}; +read_resource_uri(Path, Uri) -> + {NS, Rest} = case binary:split(Path, <<":">>) of + [NS, R] -> {NS, R}; + [NS] -> {NS, undefined} + end, + case namespace_to_module(NS) of + {ok, Mod} -> Mod:read_uri(Rest, Uri); + error -> {error, invalid_namespace, NS} + end. + +namespace_to_module(<<"tasks">>) -> namespace_to_module(tasks); +namespace_to_module(<<"names">>) -> namespace_to_module(names); +namespace_to_module(tasks) -> {ok, dreki_tasks}; +namespace_to_module(names) -> {ok, dreki_names}; +namespace_to_module(_) -> error. + +internal_subdomain() -> + strip_domain(internal_domain(), root_domain()). + +root_path() -> + join(<<":">>, [<<"dreki">>, root_domain(), internal_subdomain()]). + +strip_root_path(Path) -> + Root = root_path(), + binary:replace(Path, <>, <<"">>). + +put_root_path(Path) -> + join(<<":">>, [root_path(), Path]). + +path() -> + HierJ = lists:reverse(hierarchy()), + Components = [<<"dreki">>, root_domain(), internal_subdomain()] ++ HierJ, + join(<<":">>, Components). + +hierarchy() -> + binary:split(strip_domain(domain(), internal_domain()), <<".">>, [global]). + +parent() -> + case hierarchy() of + [_, Parent | _] -> Parent; + [Parent] -> Parent + end. + +me() -> + [Me | _] = hierarchy(), + Me. + +get_env(Key) -> + {ok, Val} = application:get_env(dreki, Key), + Val. + +to_map() -> + #{domain => #{root => root_domain(), internal => internal_domain(), self => domain()}, + path => path(), + root_path => root_path(), + hiearchy => lists:reverse(hierarchy()), + parent => parent(), + me => me()}. + +strip_domain(FQDN, Parent) -> + binary:replace(FQDN, <<".", Parent/binary>>, <<"">>). + +join(_Separator, []) -> + <<>>; +join(Separator, [H|T]) -> + lists:foldl(fun (Value, Acc) -> <> end, H, T). diff --git a/apps/dreki/src/world/dreki_world_dns.erl b/apps/dreki/src/world/dreki_world_dns.erl new file mode 100644 index 0000000..058c9db --- /dev/null +++ b/apps/dreki/src/world/dreki_world_dns.erl @@ -0,0 +1,162 @@ +-module(dreki_world_dns). + +-export([start/0, graph/0, node_ips/1, node/0, parents/1, nodes/0, regions/0, vertex/1, node_params/1, node_param/2, as_map/0]). +-export([get_path/2, in_neighbours/1, out_neighbours/1]). + +get_path(V1, V2) -> + digraph:get_path(graph(), V1, V2). + +in_neighbours(V) -> + digraph:in_neighbours(graph(), V). + +out_neighbours(V) -> + digraph:out_neighbours(graph(), V). + +as_map() -> + Vertices = lists:foldr(fun (V = {Type, Key}, Acc) -> + {_, Label} = digraph:vertex(graph(), V), + BType = atom_to_binary(Type), + Node = <>, + [#{node => Node, type => Type, name => Key, data => Label} | Acc] + end, [], digraph:vertices(graph())), + Edges = lists:foldr(fun (E, Acc) -> + {_, {Ft, Fn}, {Tt, Tn}, Label} = digraph:edge(graph(), E), + BFt = atom_to_binary(Ft), + BTt = atom_to_binary(Tt), + Fk = <>, + Tk = <>, + [#{from => Fk, to => Tk, data => Label} | Acc] + end, [], digraph:edges(graph())), + #{vertices => Vertices, edges => Edges}. + +vertex(V) -> + digraph:vertex(graph(), V). + +nodes() -> + [N || V = {node, N} <- digraph_utils:topsort(graph())]. + +regions() -> + vertices(region). + +vertices(Type) -> + [V || V = {Type, _} <- digraph_utils:topsort(graph())]. + +node() -> + {node, dreki_world:domain()}. + +node_params(N) -> + case digraph:vertex(graph(), {node, N}) of + {_, Params} -> {ok, Params}; + _ -> {error, no_such_node} + end. + +node_param(N, Key) -> + case node_params(N) of + {ok, P} -> {ok, maps:get(Key, P)}; + Err -> Err + end. + +parents(V) -> + digraph:in_neighbours(graph(), V). + +node_ips(N) -> + case digraph:vertex(graph(), {node, N}) of + {{node, N}, #{srvs := SRVs}} -> + IPs = [ {I,Port} || #{name := Name, port := Port} <- SRVs, + T <- [a, aaaa], + {ok, {_,_,_,_,_,Ip}} <- [inet_res:getbyname(binary_to_list(Name), T)], + I <- Ip], + {ok, lists:flatten(IPs)}; + Err -> + {error, {no_such_node, N, Err}} + end. + +start() -> + {ok, Graph, Errs} = build(), + persistent_term:put({?MODULE, graph}, Graph), + {ok, Errs}. + +graph() -> + persistent_term:get({?MODULE, graph}). + +build() -> + Root = dreki_world:internal_domain(), + Host = sd_dns(Root), + case inet_res:getbyname(Host, srv) of + {ok, {hostent, _Host, [], srv, _, SRVs}} -> + Graph = digraph:new([acyclic]), + {Name, NameErrs} = read_txt(name_dns(Root), Root), + V = {root, Root}, + digraph:add_vertex(Graph, V, #{display_name => Name}), + Errs = collect_sd_srvs(SRVs, V, Graph, NameErrs), + {ok, Graph, lists:flatten(Errs)}; + {error, DNSErr} when is_atom(DNSErr) -> + {error, #{error => "world_dns_error", dns_error => DNSErr, host => Host}} + end. + +expand_sd_srv(Host, Parent, Graph) -> + NodeHost = node_dns(Host), + {Vn, Acc0} = case inet_res:getbyname(NodeHost, srv) of + {ok, {hostent, _, _, srv, _, NSRVs}} -> + Targets = lists:foldr(fun ({Priority, Weight, Port, Name}, Acc) -> + [#{name => list_to_binary(Name), port => Port, priority => Priority, weight => Weight} | Acc] + end, [], NSRVs), + {NName, NameErrs} = read_txt(name_dns(Host), name(Host)), + {NodeName, NameErrs2} = read_txt(node_name_dns(Host), <<"dreki@", Host/binary>>), + Nv = {node, Host}, + digraph:add_vertex(Graph, Nv, #{display_name => NName, srvs => Targets, node_name => binary_to_atom(NodeName)}), + digraph:add_edge(Graph, Parent, Nv, #{}), + {Nv, [] ++ NameErrs ++ NameErrs2}; + {error, nxdomain} -> {undefined, []}; + {error, VDNSErr} when is_atom(VDNSErr) -> + logger:log(error, #{dns_error => VDNSErr, host => NodeHost}), + {undefined, [{error, #{error => "world_dns_error", dns_error => VDNSErr, host => NodeHost}}]} + end, + SdHost = sd_dns(Host), + Acc1 = case inet_res:getbyname(SdHost, srv) of + {ok, {hostent, _SdHost, [], srv, _, SSRVs}} -> + {RName, RNameErrs} = read_txt(name_dns(Host), name(Host)), + V = {region, Host}, + digraph:add_vertex(Graph, V, #{display_name => RName}), + case Vn of + undefined -> digraph:add_edge(Graph, Parent, V, #{}); + Vn -> digraph:add_edge(Graph, Vn, V, #{}) + end, + collect_sd_srvs(SSRVs, V, Graph, RNameErrs); + {error, nxdomain} -> []; + {error, DNSErr} when is_atom(DNSErr) -> + logger:log(error, #{dns_error => DNSErr, host => SdHost}), + [{error, #{error => "world_dns_error", dns_error => DNSErr, host => SdHost}}] + end, + [Acc0, Acc1]. + +collect_sd_srvs([], _, _Graph, Acc) -> Acc; +collect_sd_srvs([{0, 0, 1337, Entry} | Rest], Parent, Graph, Acc) -> + collect_sd_srvs(Rest, Parent, Graph, [expand_sd_srv(list_to_binary(Entry), Parent, Graph) | Acc]). + +read_txt(Host, Default) -> + case inet_res:getbyname(Host, txt) of + {error, nxdomain} -> {Default, []}; + {ok,{hostent, _, _, _, _, Lines}} -> {list_to_binary(Lines), []}; + {error, DNSErr} when is_atom(DNSErr) -> + logger:log(error, #{dns_error => DNSErr, host => Host}), + {Default, [#{error => "world_dns_error", dns_error => DNSErr, host => Host}]} + end. + +sd_dns(Domain) -> dnsname(<<"_dreki">>, Domain). +node_dns(Domain) -> dnsname(<<"_node._dreki">>, Domain). +name_dns(Domain) -> dnsname(<<"_name._dreki">>, Domain). +node_name_dns(Domain) -> dnsname(<<"_name._node._dreki">>, Domain). + +dnsname(Prefix, Domain) when is_list(Prefix) -> + dnsname(list_to_binary(Prefix), Domain); +dnsname(Prefix, Domain) when is_list(Domain) -> + dnsname(Prefix, list_to_binary(Domain)); +dnsname(Prefix, Domain) -> + Full = <>, + binary:bin_to_list(Full). + +name(Host) when is_list(Host) -> + name(list_to_binary(Host)); +name(Host) -> + hd(binary:split(Host, <<".">>)). diff --git a/apps/dreki/src/world/dreki_world_plum_events.erl b/apps/dreki/src/world/dreki_world_plum_events.erl new file mode 100644 index 0000000..b32ae09 --- /dev/null +++ b/apps/dreki/src/world/dreki_world_plum_events.erl @@ -0,0 +1,17 @@ +-module(dreki_world_plum_events). +-behaviour(gen_event). +-export([init/1, handle_call/2, handle_event/2, terminate/2]). + +init([Server]) -> + {ok, Server}. + +handle_call(_, Server) -> + {ok, error, Server}. + +handle_event(Event, Server) -> + logger:info("plum_event: ~p", [Event]), + dreki_world_server:send_event(Server, {plum_events, Event}), + {ok, Server}. + +terminate(_, Server) -> + ok. diff --git a/apps/dreki/src/world/dreki_world_server.erl b/apps/dreki/src/world/dreki_world_server.erl new file mode 100644 index 0000000..2bc41ed --- /dev/null +++ b/apps/dreki/src/world/dreki_world_server.erl @@ -0,0 +1,63 @@ +-module(dreki_world_server). +-behaviour(partisan_gen_fsm). +-include_lib("kernel/include/logger.hrl"). + +-export([start_link/1, send_event/2]). +-export([init/1]). +-export([setup/2]). +-export([wait/2]). + +-record(data, { + path=undefined +}). + +start_link(Args) -> + partisan_gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). + +send_event(Name, Event) -> + partisan_gen_fsm:send_event(Name, Event). + +init(Args) -> + ok = plum_db_events:add_handler(dreki_world_plum_events, [?MODULE]), + Data = #data{path = dreki_world:path()}, + ok = setup_autojoin(), + todo = setup_connect_nearest(), + ok = dreki_node:ensure_local_node(), + {ok, setup, Data, 0}. + +setup(timeout, Data) -> + ?LOG_INFO("world_server: setup done"), + {next_state, wait, Data}. + +setup_autojoin() -> + case application:get_env(dreki, autojoin, undefined) of + undefined -> ok; + List -> [dreki_peer_service:join(N) || N <- List] + end, + ok. + +setup_connect_nearest() -> + case dreki_peer_service:connect_nearest_dns() of + ok -> ok; + todo -> todo; + error -> + logger:error("Node did not join anyone from the world!!"), + error + end. + +wait({plum_events, {object_update, {Prefix, Key}, Object}}, Data) -> + process_object_update(Prefix, Key, Object, Data), + {next_state, wait, Data}; +wait(Event, Data) -> + ?LOG_INFO("world_server wait event: ~p", [Event]), + {next_state, wait, Data}. + +process_object_update({nodes, Node}, Key, _Obj, #data{path = Node}) -> + ?LOG_INFO("My node has been updated !"), + ok; +process_object_update({nodes, Node}, Key, _Obj, _Data) -> + ?LOG_INFO("Node update ~p ~p", [Node, Key]), + ok; +process_object_update(Prefix, Key, _Obj, _Data) -> + ?LOG_INFO("Metadata update ~p ~p", [Prefix, Key]), + ok. diff --git a/apps/dreki/src/world/dreki_world_store.erl b/apps/dreki/src/world/dreki_world_store.erl new file mode 100644 index 0000000..050efd0 --- /dev/null +++ b/apps/dreki/src/world/dreki_world_store.erl @@ -0,0 +1,48 @@ +-module(dreki_world_store). +-include("dreki.hrl"). +-behaviour(dradis_store_backend). + +%% Types + +-record(store, {}). +-type db() :: #store{}. +-type args() :: #{}. + +-export([start/0, start/5, checkout/1, checkin/1, stop/0, stop/1]). +-export([valid_store/5]). +-export([list/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). + +valid_store(_Namespace, Location, _Name, _NSMod, _Args) -> + case Location =:= dreki_world:root_path() of + true -> ok; + false -> {error, {dreki_world_store_invalid_location, Location, dreki_world:root_path()}} + end. + +start() -> ok. +start(_, _, _, _, _) -> {ok, dreki_world_store}. + +checkout(_) -> {ok, #store{}}. +checkin(_) -> ok. +stop() -> ok. +stop(_) -> ok. + +list(_) -> + {error, not_implemented}. + +count(_) -> + {error, not_implemented}. + +exists(_, _) -> + {error, not_implemented}. + +get(_, _) -> + {error, not_implemented}. + +delete(_, _) -> + {error, not_implemented}. + +create(_, _) -> + {error, not_implemented}. + +update(_Tab, _) -> + {error, not_implemented}. diff --git a/apps/dreki/src/world/dreki_world_tasks.erl b/apps/dreki/src/world/dreki_world_tasks.erl new file mode 100644 index 0000000..c27a7dc --- /dev/null +++ b/apps/dreki/src/world/dreki_world_tasks.erl @@ -0,0 +1,66 @@ +-module(dreki_world_tasks). +-include("dreki.hrl"). +-define(PREFIX, {objects, 'dreki_world_tasks'}). + +%% Types +-type db() :: term(). +-type args() :: #{db_name => binary()}. + +%% storage-specific +-export([start/1, open/1, sync/1, close/1, stop/1]). +-export([all/1, count/1, exists/2, get/2, create/2, update/2, delete/2]). + +start(_) -> {ok, dreki_world_tasks}. +open(_) -> {ok, dreki_world_tasks}. +close(_) -> ok. +stop(_) -> ok. +sync(_) -> noop. + +all(_) -> + Results = plum_db:fold(fun ({_, Value}, Acc) -> + [Value | Acc] + end, [], ?PREFIX), + {ok, Results}. + +count(_) -> + Results = plum_db:fold(fun (_, Acc) -> + Acc + 1 + end, 0, {objects, 'dreki_world_tasks'}), + {ok, Results}. + +exists(T, Id) -> + case get(T, Id) of + {ok, _} -> true; + _ -> false + end. + +get(_, Id) -> + case dreki_world:get(?PREFIX, Id) of + {ok, Val} -> {ok, Val}; + not_found -> {error, {task_not_found, Id}} + end. + +create(T, Task = #dreki_task{id = Id}) -> + case get(T, Id) of + {ok, _} -> {error, {exists, Id}}; + {error, {task_not_found, _}} -> + ok = plum_db:put(?PREFIX, Id, T), + ok = dreki_world:refresh_index(Task), + get(T, Id) + end. + +update(T, Task = #dreki_task{id = Id}) -> + case get(T, Id) of + Error = {error, {task_not_found, _}} -> Error; + {ok, _OldTask} -> + ok = plum_db:put(?PREFIX, Id, T), + ok = dreki_world:refresh_index(Task), + get(T, Id) + end. + +delete(T, #dreki_task{id = Id}) -> + {error, {dreki_world_tasks, delete_not_implemented}}. + + +refresh_index(#dreki_task{uri = URI, tags = Tags, roles = Roles}) -> + dreki_world:refresh_index(tasks, #{uri => URI, tags => Tags, roles => Roles}). diff --git a/apps/dreki_web/.gitignore b/apps/dreki_web/.gitignore index 8dec94f..8946870 100644 --- a/apps/dreki_web/.gitignore +++ b/apps/dreki_web/.gitignore @@ -18,3 +18,4 @@ _build rebar3.crashdump *~ /priv/static/ +/assets/node_modules/ diff --git a/apps/dreki_web/assets/package-lock.json b/apps/dreki_web/assets/package-lock.json index c99c8b3..b6aa028 100644 --- a/apps/dreki_web/assets/package-lock.json +++ b/apps/dreki_web/assets/package-lock.json @@ -12,7 +12,8 @@ "@tailwindcss/forms": "^0.5.0", "@tailwindcss/line-clamp": "^0.3.1", "@tailwindcss/typography": "^0.5.2", - "crossfilter": "^1.3.12" + "crossfilter": "^1.3.12", + "xterm": "^4.18.0" }, "devDependencies": { "@hotwired/stimulus": "^3.0.1", @@ -2063,6 +2064,11 @@ "node": ">=0.4" } }, + "node_modules/xterm": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/xterm/-/xterm-4.18.0.tgz", + "integrity": "sha512-JQoc1S0dti6SQfI0bK1AZvGnAxH4MVw45ZPFSO6FHTInAiau3Ix77fSxNx3mX4eh9OL4AYa8+4C8f5UvnSfppQ==" + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -3567,6 +3573,11 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "xterm": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/xterm/-/xterm-4.18.0.tgz", + "integrity": "sha512-JQoc1S0dti6SQfI0bK1AZvGnAxH4MVw45ZPFSO6FHTInAiau3Ix77fSxNx3mX4eh9OL4AYa8+4C8f5UvnSfppQ==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/apps/dreki_web/assets/package.json b/apps/dreki_web/assets/package.json index 07072c5..1aa07ff 100644 --- a/apps/dreki_web/assets/package.json +++ b/apps/dreki_web/assets/package.json @@ -27,6 +27,7 @@ "@tailwindcss/forms": "^0.5.0", "@tailwindcss/line-clamp": "^0.3.1", "@tailwindcss/typography": "^0.5.2", - "crossfilter": "^1.3.12" + "crossfilter": "^1.3.12", + "xterm": "^4.18.0" } } diff --git a/apps/dreki_web/rebar.config b/apps/dreki_web/rebar.config index b16060a..e439af6 100644 --- a/apps/dreki_web/rebar.config +++ b/apps/dreki_web/rebar.config @@ -5,7 +5,8 @@ {erlydtl, "0.14.0"}, {oauth2c, {git, "https://github.com/kivra/oauth2_client", {branch, "master"}}}, {cowboy_telemetry, "~> 0.4.0"}, - {opentelemetry_cowboy, "~> 0.1.0"} + {opentelemetry_cowboy, "~> 0.1.0"}, + {prometheus_cowboy, "0.1.8"} ]}. {plugins, [ diff --git a/apps/dreki_web/src/admin/dreki_web_admin_tasks.erl b/apps/dreki_web/src/admin/dreki_web_admin_tasks.erl new file mode 100644 index 0000000..adc2689 --- /dev/null +++ b/apps/dreki_web/src/admin/dreki_web_admin_tasks.erl @@ -0,0 +1,30 @@ +-module(dreki_web_admin_tasks). +-behaviour(cowboy_rest). +-export([init/2]). +-export([allowed_methods/2]). +-export([content_types_accepted/2]). +-export([content_types_provided/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +allowed_methods(Req, State) -> + {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. + +content_types_accepted(Req, State) -> + dreki_web:content_types_accepted(Req, State). + +content_types_provided(Req, State) -> + dreki_web:content_types_provided(Req, State). + +list(Req, State, Format) -> + {ok, Db} = dreki_tasks:open(), + {ok, Tasks} = dreki_tasks:all(Db), + MTasks = lists:foldl(fun (T, Acc) -> [dreki_task:to_map(T) | Acc] end, [], Tasks), + dreki_web:reply(Req, 200, #{error => false, tasks => MTasks}, [], Format). + +to_json(Req, State) -> + list(Req, State, json). + +to_yaml(Req, State) -> + list(Req, State, yaml). diff --git a/apps/dreki_web/src/admin/dreki_web_admin_world.erl b/apps/dreki_web/src/admin/dreki_web_admin_world.erl new file mode 100644 index 0000000..1ceaaee --- /dev/null +++ b/apps/dreki_web/src/admin/dreki_web_admin_world.erl @@ -0,0 +1,28 @@ +-module(dreki_web_admin_world). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req, index) -> + Json = #{<<"data">> => #{<<"service">> => <<"dreki">>}}, + {ok, dreki_web:reply_json(Req, 200, Json), undefined}; + +init(Req, graph) -> + World = dreki_world_dns:as_map(), + Json = #{<<"data">> => World}, + {ok, dreki_web:reply_json(Req, 200, Json), undefined}; + +init(Req, {graph, dot}) -> + World = dreki_world_dns:as_map(), + Vertices = [dot_format_vertex(N) || N <- maps:get(vertices, World)], + {ok, Dot} = world_graph_dot_dtl:render([{vertices, Vertices}, {edges, maps:get(edges, World)}]), + cowboy_req:reply(200, #{<<"content-type">> => <<"text/vnd.graphviz">>}, Dot, Req); + +init(Req, _) -> + dreki_web_error:init(Req, #{code => 404, status => "Not found"}). + +dot_format_vertex(V = #{type := node}) -> + V#{shape => <<"box">>, color => <<"#c2410c">>, class => <<"dreki-world-graph-node">>}; +dot_format_vertex(V = #{type := root}) -> + V#{shape => <<"polygon">>, color => <<"#a16207">>, class => <<"dreki-world-graph-root">>}; +dot_format_vertex(V = #{type := region}) -> + V#{shape => <<"egg">>, color => <<"#4d7c0f">>, class => <<"dreki-world-graph-region">>}. diff --git a/apps/dreki_web/src/api/dreki_web_index.erl b/apps/dreki_web/src/api/dreki_web_index.erl new file mode 100644 index 0000000..2ed8a38 --- /dev/null +++ b/apps/dreki_web/src/api/dreki_web_index.erl @@ -0,0 +1,9 @@ +-module(dreki_web_index). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req = #{method := <<"GET">>}, _) -> + Json = #{<<"error">> => false, <<"service">> => <<"dreki">>}, + {ok, dreki_web:reply_json(Req, 200, Json), undefined}; +init(Req, _) -> + dreki_web_error:init(Req, #{code => 400, status => "Bad request"}). diff --git a/apps/dreki_web/src/api/dreki_web_task.erl b/apps/dreki_web/src/api/dreki_web_task.erl new file mode 100644 index 0000000..fdcf9cd --- /dev/null +++ b/apps/dreki_web/src/api/dreki_web_task.erl @@ -0,0 +1,21 @@ +-module(dreki_web_task). +-behaviour(cowboy_handler). +-behaviour(cowboy_rest). +-export([init/2]). +-export([allowed_methods/2]). +-export([content_types_accepted/2]). +-export([content_types_provided/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +allowed_methods(Req, State) -> + {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. + +content_types_accepted(Req, State) -> + dreki_web:content_types_accepted(Req, State). + +content_types_provided(Req, State) -> + dreki_web:content_types_provided(Req, State). + + diff --git a/apps/dreki_web/src/cowboy_access_log_h.erl b/apps/dreki_web/src/cowboy_access_log_h.erl new file mode 100644 index 0000000..433d706 --- /dev/null +++ b/apps/dreki_web/src/cowboy_access_log_h.erl @@ -0,0 +1,238 @@ +-module(cowboy_access_log_h). +-behaviour(cowboy_stream). + +-dialyzer(no_undefined_callbacks). + +-type extra_info_fun() :: fun((cowboy_req:req()) -> #{atom() => term()}). +-export_type([extra_info_fun/0]). + +%% API exports + +-export([set_extra_info_fun/2]). + +%% callback exports + +-export([init/3]). +-export([data/4]). +-export([info/3]). +-export([terminate/3]). +-export([early_error/5]). + +-type state() :: #{ + next := any(), + req := cowboy_req:req(), + meta := #{started_at => genlib_time:ts()}, + ext_fun := extra_info_fun() +}. + +%% API + +-spec set_extra_info_fun(extra_info_fun(), cowboy:opts()) + -> cowboy:opts(). +set_extra_info_fun(Fun, Opts) when is_function(Fun, 1) -> + Opts#{extra_info_fun => Fun}. + +%% callbacks + +-spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts()) + -> {cowboy_stream:commands(), state()}. +init(StreamID, Req, Opts) -> + State = make_state(Req, Opts), + {Commands0, Next} = cowboy_stream:init(StreamID, Req, Opts), + {Commands0, State#{next => Next}}. + +-spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State) + -> {cowboy_stream:commands(), State} when State::state(). +data(StreamID, IsFin, Data, #{next := Next0} = State) -> + {Commands0, Next} = cowboy_stream:data(StreamID, IsFin, Data, Next0), + {Commands0, State#{next => Next}}. + +-spec info(cowboy_stream:streamid(), any(), State) + -> {cowboy_stream:commands(), State} when State::state(). +info(StreamID, {IsResponse, Code, Headers, _} = Info, #{req := Req, next := Next0} = State) when + IsResponse == response; + IsResponse == error_response +-> + _ = log_access_safe(Code, Headers, State, get_request_body_length(Req)), + {Commands0, Next} = cowboy_stream:info(StreamID, Info, Next0), + {Commands0, State#{next => Next}}; +info(StreamID, Info, #{next := Next0} = State) -> + {Commands0, Next} = cowboy_stream:info(StreamID, Info, Next0), + {Commands0, State#{next => Next}}. + +-spec terminate(cowboy_stream:streamid(), cowboy_stream:reason(), state()) -> any(). +terminate(StreamID, Reason, #{next := Next}) -> + cowboy_stream:terminate(StreamID, Reason, Next). + +-spec early_error(cowboy_stream:streamid(), cowboy_stream:reason(), + cowboy_stream:partial_req(), Resp, cowboy:opts()) -> Resp + when Resp::cowboy_stream:resp_command(). + +%% NOTE: in early_error cowboy uses PartialReq, a cowboy_req:req() - like structure +%% for more info see https://ninenines.eu/docs/en/cowboy/2.7/manual/cowboy_stream/#_callbacks + +early_error(StreamID, Reason, PartialReq, {_, Code, Headers, _} = Resp, Opts) -> + State = make_state(PartialReq, Opts), + _ = log_access_safe(Code, Headers, State, undefined), + cowboy_stream:early_error(StreamID, Reason, PartialReq, Resp, State). + +%% private functions + +log_access_safe(Code, Headers, #{req := Req} = State, ReqBodyLength) -> + try + logger:log(info, prepare_meta(Code, Headers, State, ReqBodyLength)), + Req + catch + Class:Reason:Stacktrace -> + Stack = genlib_format:format_stacktrace(Stacktrace, [newlines]), + _ = logger:error( + "Log access failed for: [~p, ~p, ~p]~nwith: ~p:~p~nstacktrace: ~ts", + [Code, Headers, Req, Class, Reason, Stack] + ), + Req + end. + +get_process_meta() -> + case logger:get_process_metadata() of + undefined -> + #{}; + Meta -> + Meta + end. + +% domain field specifies the functional area that send log event +% as we want to save logs from this app in a separate file, +% we can easily filter logs by their domain using OTP filter functions. +prepare_meta(Code, Headers, #{req := Req, meta:= Meta0, ext_fun := F}, ReqBodyLength) -> + AccessMeta = genlib_map:compact(#{ + domain => [cowboy_access_log], + status => Code, + remote_addr => get_remote_addr(Req), + peer_addr => get_peer_addr(Req), + request_method => cowboy_req:method(Req), + request_path => cowboy_req:path(Req), + request_length => ReqBodyLength, + response_length => get_response_len(Headers), + request_time => get_request_duration(Meta0), + trace_id => maps:get(<<"trace-id">>, Req, <<"no-trace">>), + 'http_x-request-id' => cowboy_req:header(<<"x-request-id">>, Req, undefined) + }), + AccessMeta1 = maps:merge(get_process_meta(), AccessMeta), + maps:merge(F(Req), AccessMeta1). + +get_request_body_length(Req) -> + case cowboy_req:has_body(Req) of + false -> undefined; + true -> cowboy_req:body_length(Req) + end. + +get_peer_addr(Req) -> + {IP, _Port} = cowboy_req:peer(Req), + genlib:to_binary(inet:ntoa(IP)). + +get_remote_addr(Req) -> + case determine_remote_addr(Req) of + {ok, RemoteAddr} -> + genlib:to_binary(inet:ntoa(RemoteAddr)); + _ -> + undefined + end. + +determine_remote_addr(Req) -> + Peer = cowboy_req:peer(Req), + Value = cowboy_req:header(<<"x-forwarded-for">>, Req), + determine_remote_addr_from_header(Value, Peer). + +determine_remote_addr_from_header(undefined, {IP, _Port}) -> + % undefined, assuming no proxies were involved + {ok, IP}; +determine_remote_addr_from_header(Value, _Peer) when is_binary(Value) -> + ClientPeer = string:strip(binary_to_list(Value)), + case string:tokens(ClientPeer, ", ") of + [ClientIP | _Proxies] -> + inet:parse_strict_address(ClientIP); + _ -> + {error, malformed} + end. + +get_request_duration(Meta) -> + case maps:get(started_at, Meta, undefined) of + undefined -> + undefined; + StartTime -> + (genlib_time:ticks() - StartTime) / 1000000 + end. + +get_response_len(Headers) -> + case maps:get(<<"content-length">>, Headers, undefined) of + undefined -> + undefined; + Len -> + genlib:to_int(Len) + end. + +make_state(Req, Opts) -> + ExtFun = make_ext_fun(Opts), + set_meta(#{req => Req, ext_fun => ExtFun}). + +set_meta(State) -> + State#{meta => #{started_at => genlib_time:ticks()}}. + +make_ext_fun(Opts) -> + maps:get(extra_info_fun, Opts, fun(_Req) -> #{} end). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +-spec test() -> _. + +-spec filter_meta_test() -> _. +filter_meta_test() -> + Req = #{ + pid => self(), + peer => {{42, 42, 42, 42}, 4242}, + method => <<"GET">>, + path => <<>>, + qs => <<>>, + version => 'HTTP/1.1', + headers => #{}, + host => <<>>, + port => undefined, + has_body => true + }, + State = make_state(Req, #{}), + #{ + request_method := <<"GET">>, + request_path := <<>>, + request_time := _, + response_length := 33, + request_length := 100, + peer_addr := <<"42.42.42.42">>, + status := 200 + } = prepare_meta(200, #{<<"content-length">> => <<"33">>}, State, 100). + +-spec filter_meta_for_error_test() -> _. +filter_meta_for_error_test() -> + Req = #{ + pid => self(), + peer => {{42, 42, 42, 42}, 4242}, + method => <<"GET">>, + path => <<>>, + qs => <<>>, + version => 'HTTP/1.1', + headers => #{}, + host => <<>>, + port => undefined, + has_body => true + }, + State = make_state(Req, #{}), + #{ + peer_addr := <<"42.42.42.42">>, + remote_addr := <<"42.42.42.42">>, + request_method := <<"GET">>, + request_path := <<>>, + request_time := _, + status := 400 + } = prepare_meta(400, #{}, State, undefined). + +-endif. diff --git a/apps/dreki_web/src/dreki_web.app.src b/apps/dreki_web/src/dreki_web.app.src index cf731fc..4122e3b 100644 --- a/apps/dreki_web/src/dreki_web.app.src +++ b/apps/dreki_web/src/dreki_web.app.src @@ -10,7 +10,8 @@ cowboy, trails, cowboy_telemetry, - opentelemetry_cowboy + opentelemetry_cowboy, + prometheus_cowboy ]}, {env,[]}, {modules, []}, diff --git a/apps/dreki_web/src/dreki_web_admin_tasks.erl b/apps/dreki_web/src/dreki_web_admin_tasks.erl deleted file mode 100644 index adc2689..0000000 --- a/apps/dreki_web/src/dreki_web_admin_tasks.erl +++ /dev/null @@ -1,30 +0,0 @@ --module(dreki_web_admin_tasks). --behaviour(cowboy_rest). --export([init/2]). --export([allowed_methods/2]). --export([content_types_accepted/2]). --export([content_types_provided/2]). - -init(Req, State) -> - {cowboy_rest, Req, State}. - -allowed_methods(Req, State) -> - {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. - -content_types_accepted(Req, State) -> - dreki_web:content_types_accepted(Req, State). - -content_types_provided(Req, State) -> - dreki_web:content_types_provided(Req, State). - -list(Req, State, Format) -> - {ok, Db} = dreki_tasks:open(), - {ok, Tasks} = dreki_tasks:all(Db), - MTasks = lists:foldl(fun (T, Acc) -> [dreki_task:to_map(T) | Acc] end, [], Tasks), - dreki_web:reply(Req, 200, #{error => false, tasks => MTasks}, [], Format). - -to_json(Req, State) -> - list(Req, State, json). - -to_yaml(Req, State) -> - list(Req, State, yaml). diff --git a/apps/dreki_web/src/dreki_web_admin_world.erl b/apps/dreki_web/src/dreki_web_admin_world.erl deleted file mode 100644 index 1ceaaee..0000000 --- a/apps/dreki_web/src/dreki_web_admin_world.erl +++ /dev/null @@ -1,28 +0,0 @@ --module(dreki_web_admin_world). --behaviour(cowboy_handler). --export([init/2]). - -init(Req, index) -> - Json = #{<<"data">> => #{<<"service">> => <<"dreki">>}}, - {ok, dreki_web:reply_json(Req, 200, Json), undefined}; - -init(Req, graph) -> - World = dreki_world_dns:as_map(), - Json = #{<<"data">> => World}, - {ok, dreki_web:reply_json(Req, 200, Json), undefined}; - -init(Req, {graph, dot}) -> - World = dreki_world_dns:as_map(), - Vertices = [dot_format_vertex(N) || N <- maps:get(vertices, World)], - {ok, Dot} = world_graph_dot_dtl:render([{vertices, Vertices}, {edges, maps:get(edges, World)}]), - cowboy_req:reply(200, #{<<"content-type">> => <<"text/vnd.graphviz">>}, Dot, Req); - -init(Req, _) -> - dreki_web_error:init(Req, #{code => 404, status => "Not found"}). - -dot_format_vertex(V = #{type := node}) -> - V#{shape => <<"box">>, color => <<"#c2410c">>, class => <<"dreki-world-graph-node">>}; -dot_format_vertex(V = #{type := root}) -> - V#{shape => <<"polygon">>, color => <<"#a16207">>, class => <<"dreki-world-graph-root">>}; -dot_format_vertex(V = #{type := region}) -> - V#{shape => <<"egg">>, color => <<"#4d7c0f">>, class => <<"dreki-world-graph-region">>}. diff --git a/apps/dreki_web/src/dreki_web_app.erl b/apps/dreki_web/src/dreki_web_app.erl index 5b6454e..5b3c1a0 100644 --- a/apps/dreki_web/src/dreki_web_app.erl +++ b/apps/dreki_web/src/dreki_web_app.erl @@ -13,8 +13,12 @@ start(_StartType, _StartArgs) -> Config = application:get_all_env(dreki_web), Transport = proplists:get_value(transport, Config), CowboyEnv = #{ - middlewares => [dreki_web_auth, cowboy_router, cowboy_handler], - stream_handlers => [cowboy_telemetry_h, cowboy_stream_h], + middlewares => [dreki_web_auth, cowboy_router, dreki_web_handler], + stream_handlers => [cowboy_telemetry_h, + cowboy_access_log_h, + cowboy_metrics_h, + cowboy_stream_h], + metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, env => #{ dispatch => routes() } @@ -35,7 +39,8 @@ routes() -> Trails = [ {"/", dreki_web_index, undefined}, {"/static/[...]", cowboy_static, - {priv_dir, dreki_web, "static", [{mimetypes, dreki_web, detect_web_mimetype}]}}, + {priv_dir, dreki_web, "static", [{mimetypes, dreki_web, detect_web_mimetype}]}}, + {"/metrics/[:registry]", prometheus_cowboy2_handler, []}, %% API {"/api/tasks/:id", dreki_web_task, undefined}, @@ -59,6 +64,7 @@ routes() -> {"/admin/:location/:namespace/:directory", dreki_web_ui_stores, undefined}, {"/admin/:location/:namespace/:directory/_/:action", dreki_web_ui_stores, action}, {"/admin/:location/:namespace/:directory/:id", dreki_web_ui_stores, undefined}, + {"/admin/:location/:namespace/:directory/:id/_/:action", dreki_web_ui_stores, action}, {"/admin/[...]", dreki_web_ui_error, #{code => 404, status => <<"Not found">>}}, diff --git a/apps/dreki_web/src/dreki_web_handler.erl b/apps/dreki_web/src/dreki_web_handler.erl new file mode 100644 index 0000000..a30fd1d --- /dev/null +++ b/apps/dreki_web/src/dreki_web_handler.erl @@ -0,0 +1,99 @@ +%% Copyright (c) 2011-2017, Loïc Hoguin +%% +%% Permission to use, copy, modify, and/or distribute this software for any +%% purpose with or without fee is hereby granted, provided that the above +%% copyright notice and this permission notice appear in all copies. +%% +%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +%% Handler middleware. +%% +%% Execute the handler given by the handler and handler_opts +%% environment values. The result of this execution is added to the +%% environment under the result value. +-module(dreki_web_handler). +-include_lib("opentelemetry_api/include/otel_tracer.hrl"). +-behaviour(cowboy_middleware). + +-export([execute/2]). +-export([terminate/4]). + +-callback init(Req, any()) + -> {ok | module(), Req, any()} + | {module(), Req, any(), any()} + when Req::cowboy_req:req(). + +-callback terminate(any(), map(), any()) -> ok. +-optional_callbacks([terminate/3]). + +-spec execute(Req, Env) -> {ok, Req, Env} + when Req::cowboy_req:req(), Env::cowboy_middleware:env(). +execute(Req, Env=#{handler := Handler}) -> + Headers = maps:get(headers, Req), + otel_propagator_text_map:extract(maps:to_list(Headers)), + HandlerB = atom_to_binary(Handler), + Method = maps:get(method, Req), + Attributes = [ + {'http.host', maps:get(host, Req)}, + {'http.host.port', maps:get(port, Req)}, + {'http.method', Method}, + {'http.scheme', maps:get(scheme, Req)}, + {'http.target', maps:get(path, Req)}, + {'http.user_agent', maps:get(<<"user-agent">>, Headers, <<"">>)} + ], + SpanName = iolist_to_binary([<<"HTTP ">>, Method, <<" ">>, HandlerB]), + logger:debug("SpanName: ~p", [SpanName]), + ?with_span(SpanName, #{}, fun(Ctx) -> execute_(Req, Env, Ctx) end). + +execute_(Req0, Env=#{handler := Handler, handler_opts := HandlerOpts}, Ctx) -> + TraceId = otel_span:hex_trace_id(otel_tracer:current_span_ctx()), + Req = Req0#{<<"trace-id">> => TraceId}, + try Handler:init(Req, HandlerOpts) of + {ok, Req2, State} -> + Headers0 = maps:get(headers, Req2), + Headers = otel_propagator_text_map:inject(Headers0, + fun(Headers, Key, Value) -> + maps:put(Key, Value, Headers) + end), + Req3 = maps:put(headers, Headers, Req2), + Result = terminate(normal, Req3, State, Handler), + {ok, Req3, Env#{result => Result, trace_id => TraceId}}; + {Mod, Req2, State} -> + ?add_event(<<"HTTP_UPGRADE">>, #{module => Mod}), + Mod:upgrade(Req2, Env, Handler, State); + {Mod, Req2, State, Opts} -> + ?add_event(<<"HTTP_UPGRADE">>, #{module => Mod}), + Mod:upgrade(Req2, Env, Handler, State, Opts) + catch Class:Reason:Stacktrace -> + ?set_status(error, iolist_to_binary([atom_to_binary(Class)])), + render_error(Class, Reason, Stacktrace, Handler, Req, Env, Ctx) + end. + +render_error(Class, Reason, Stacktrace, Handler, Req, Env, Ctx) -> + ReasonS = lists:flatten(io_lib:format("~p", [Reason])), + Assigns = [ + {dreki_node, node()}, + {site_title, "Dreki"}, + {class, atom_to_binary(Class)}, + {reason, ReasonS}, + {stacktrace, [lists:flatten(io_lib:format("~p", [Line])) || Line <- Stacktrace]}, + {trace_id, maps:get(<<"trace-id">>, Req, <<"no-trace">>)} + ], + logger:error(#{app => dreki_web, handler => Handler, error => {Class, Reason, Stacktrace}}), + {ok, Html} = crash_dtl:render(Assigns), + cowboy_req:reply(500, #{<<"content-type">> => <<"text/html">>}, Html, Req). + +-spec terminate(any(), Req | undefined, any(), module()) -> ok when Req::cowboy_req:req(). +terminate(Reason, Req, State, Handler) -> + case erlang:function_exported(Handler, terminate, 3) of + true -> + Handler:terminate(Reason, Req, State); + false -> + ok + end. diff --git a/apps/dreki_web/src/dreki_web_index.erl b/apps/dreki_web/src/dreki_web_index.erl deleted file mode 100644 index 2ed8a38..0000000 --- a/apps/dreki_web/src/dreki_web_index.erl +++ /dev/null @@ -1,9 +0,0 @@ --module(dreki_web_index). --behaviour(cowboy_handler). --export([init/2]). - -init(Req = #{method := <<"GET">>}, _) -> - Json = #{<<"error">> => false, <<"service">> => <<"dreki">>}, - {ok, dreki_web:reply_json(Req, 200, Json), undefined}; -init(Req, _) -> - dreki_web_error:init(Req, #{code => 400, status => "Bad request"}). diff --git a/apps/dreki_web/src/dreki_web_task.erl b/apps/dreki_web/src/dreki_web_task.erl deleted file mode 100644 index fdcf9cd..0000000 --- a/apps/dreki_web/src/dreki_web_task.erl +++ /dev/null @@ -1,21 +0,0 @@ --module(dreki_web_task). --behaviour(cowboy_handler). --behaviour(cowboy_rest). --export([init/2]). --export([allowed_methods/2]). --export([content_types_accepted/2]). --export([content_types_provided/2]). - -init(Req, State) -> - {cowboy_rest, Req, State}. - -allowed_methods(Req, State) -> - {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. - -content_types_accepted(Req, State) -> - dreki_web:content_types_accepted(Req, State). - -content_types_provided(Req, State) -> - dreki_web:content_types_provided(Req, State). - - diff --git a/apps/dreki_web/src/dreki_web_ui.erl b/apps/dreki_web/src/dreki_web_ui.erl deleted file mode 100644 index d60cc6e..0000000 --- a/apps/dreki_web/src/dreki_web_ui.erl +++ /dev/null @@ -1,43 +0,0 @@ --module(dreki_web_ui). --export([render/3, reply_html/3, reply_html/4]). - -render(Req, InnerModule, Assigns0) -> - Assigns = assigns(Req, Assigns0), - {ok, InnerHtml} = InnerModule:render(Assigns), - render_layout(Req, InnerHtml, Assigns). - -reply_html(Req, Code, Html) -> - reply_html(Req, Code, Html, #{}). - -reply_html(Req, Code, Html, Headers0) -> - Headers = maps:put(<<"content-type">>, <<"text/html">>, Headers0), - cowboy_req:reply(Code, Headers, Html, Req). - -render_layout(Req, InnerHtml, Assigns) -> - {ok, Html} = layout_dtl:render([{"inner", InnerHtml} | Assigns]), - Html. - -assigns(Req, Assigns0) -> - Assigns = clean_assigns(Assigns0), - [{"site_title", "Dreki"}, - {"identity_id", maps:get(identity_id, Req)}, - {"identity", maps:get(identity, Req)}, - {"identity_name", dreki_web:identity_name(Req)}, - {"dreki_node", node()}, - {"dreki_world", dreki_world:to_map()} - | Assigns]. - -content_types_accepted(Req, State) -> - {[ - {{ <<"multipart">>, <<"form-data">>, '*'}, from_form} - ], Req, State}. - -content_types_provided(Req, State) -> - {[{{ <<"text">>, <<"html">>, '*'}, to_html}], Req, State}. - -clean_assigns(Assigns) when is_list(Assigns) -> - Assigns; -clean_assigns(Map) when is_map(Map) -> - maps:fold(fun (Key, Value, Acc) -> [{Key, clean_assigns(Value)} | Acc] end, [], Map); -clean_assigns(Other) -> - Other. diff --git a/apps/dreki_web/src/dreki_web_ui_error.erl b/apps/dreki_web/src/dreki_web_ui_error.erl deleted file mode 100644 index ccda150..0000000 --- a/apps/dreki_web/src/dreki_web_ui_error.erl +++ /dev/null @@ -1,14 +0,0 @@ --module(dreki_web_ui_error). --behaviour(cowboy_handler). --export([init/2]). - -init(Req, not_found) -> - reply(Req, 404, <<"Not Found">>, undefined); -init(Req, State = #{code := Code, status := Status}) -> - reply(Req, Code, Status, maps:get(message, State, undefined)). - -reply(Req0, Code, Status, Msg) -> - Assigns = [{"message", Msg}, {"status", Status}], - Html = dreki_web_ui:render(Req0, error_dtl, Assigns), - Req = dreki_web_ui:reply_html(Req0, Code, Html), - {ok, Req, undefined}. diff --git a/apps/dreki_web/src/dreki_web_ui_index.erl b/apps/dreki_web/src/dreki_web_ui_index.erl deleted file mode 100644 index 9f4684e..0000000 --- a/apps/dreki_web/src/dreki_web_ui_index.erl +++ /dev/null @@ -1,31 +0,0 @@ --module(dreki_web_ui_index). --behaviour(cowboy_handler). --export([init/2]). - -init(Req = #{method := <<"GET">>}, State) -> - PrettyWorld = jsone:encode(dreki_world:to_map(), [canonical_form, {space, 1}, {indent, 4}]), - LocalTasksStores = dreki_tasks:local_stores(), - {ok, Peers} = partisan_peer_service:members(), - NavTree = nav_tree(dreki_world_dns:as_map()), - Html = dreki_web_ui:render(Req, index_dtl, [ - {"page_title", "Admin UI"}, - {"dreki_world_pretty_json", PrettyWorld}, - {"dreki_local_tasks_stores", LocalTasksStores}, - {"dreki_peers", Peers}, - {"nav_tree", NavTree} - ]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; -init(Req, _) -> - dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). - -nav_tree(#{vertices := Vertices}) -> - nav_tree(Vertices, []). - -nav_tree([Region = #{type := region, name := Name, data := #{display_name := Display}} | Rest], Acc) -> - nav_tree(Rest, [Region#{href => <<"/admin/regions/", Name/binary>>, title => Display} | Acc]); -nav_tree([Node = #{type := node, name := Name, data := #{display_name := Display}} | Rest], Acc) -> - nav_tree(Rest, [Node#{href => <<"/admin/nodes/", Name/binary>>, title => Display} | Acc]); -nav_tree([#{type := Root} | Rest], Acc) -> - nav_tree(Rest, Acc); -nav_tree([], Acc) -> - Acc. diff --git a/apps/dreki_web/src/dreki_web_ui_json_form.erl b/apps/dreki_web/src/dreki_web_ui_json_form.erl deleted file mode 100644 index 49f69f8..0000000 --- a/apps/dreki_web/src/dreki_web_ui_json_form.erl +++ /dev/null @@ -1,127 +0,0 @@ --module(dreki_web_ui_json_form). --export([render_html/2]). --export([render/2]). --export([to_html/1]). - --type dreki_form() :: #{ - input => binary(), - label => binary() -}. - -render_html(Schema, Opts) -> - Abstract = render(Schema, Opts), - to_html(Abstract). - -to_html(Atom) when is_atom(Atom) -> - as_binary(Atom); -to_html(Binary) when is_binary(Binary) -> - Binary; -to_html(Abstract) -> - to_html(Abstract, []). - -to_html([{Node, Attrs} | Rest], Acc) -> - BNode = as_binary(Node), - AttrsS = attrs_to_html(Attrs), - to_html(Rest, [[<<"<">>, BNode, <<" ">>, AttrsS, <<">">>] | Acc]); -to_html([{Node, Attrs, Content} | Rest], Acc) -> - BNode = as_binary(Node), - AttrsS = attrs_to_html(Attrs), - logger:debug("Node ~p Attrs ~p Content ~p", [Node, Attrs, Content]), - to_html(Rest, [[<<"<">>, BNode, <<" ">>, AttrsS, <<">">>, to_html(Content), <<">, BNode, <<">">>] | Acc]); -to_html([], Acc) -> - lists:reverse(Acc). - -attrs_to_html(Attrs) -> attrs_to_html(Attrs, []). -attrs_to_html([{Attr, undefined} | Rest], Acc) -> - attrs_to_html(Rest, Acc); -attrs_to_html([{Attr, false} | Rest], Acc) -> - attrs_to_html(Rest, Acc); -attrs_to_html([{Attr, true} | Rest], Acc) -> - attrs_to_html(Rest, [[as_binary(Attr), <<" ">>] | Acc]); -attrs_to_html([{Attr, Num} | Rest], Acc) when is_integer(Num) -> - attrs_to_html(Rest, [[as_binary(Attr), <<"=">>, Num, <<" ">>] | Acc]); -attrs_to_html([{Attr, Value} | Rest], Acc) -> - attrs_to_html(Rest, [[as_binary(Attr), <<"=">>, <<"\"">>, as_binary(Value), <<"\"">>, <<" ">>] | Acc]); -attrs_to_html([], Acc) -> - Acc. - -render(Schema, Opts) -> - lists:reverse(maps:fold(fun (Key, Value, Acc) -> - case render_property(Key, Value, Schema, Schema, Opts) of - {ok, Html} -> [Html | Acc]; - ignore -> Acc - end - end, [], maps:get(properties, Schema))). - -render_property(Field, Config = #{enum := Enum}, Parent, Schema, Opts) -> - FormAttrs = maps:get(<<"dreki:form">>, Config, #{}), - Required = lists:member(Field, maps:get(required, Parent, [])), - Label = maps:get(label, FormAttrs, maps:get(title, Config, Field)), - InputOpts = #{required => Required, label => Label, values => Enum}, - {ok, render_input(select, Field, undefined, InputOpts, Opts)}; - -render_property(Field, Config = #{type := Type}, Parent, Schema, Opts) -> - FormAttrs = maps:get(<<"dreki:form">>, Config, #{}), - Input = maps:get(input, FormAttrs, input), - InputType = maps:get(input_type, FormAttrs, input_type(Type)), - Required = lists:member(Field, maps:get(required, Parent, [])), - Label = maps:get(label, FormAttrs, maps:get(title, Config, Field)), - InputOpts = #{required => Required, label => Label, input_type => InputType}, - {ok, render_input(Input, Field, Type, InputOpts, Opts)}; - -render_property(Field, #{'$ref' := Ref}, Parent, Schema = #{'$defs' := Defs}, Opts) -> - case maps:get(Ref, Defs, undefined) of - undefined -> logger:error("didn't get ref ~p", [Ref]); - ExpandedRef -> - logger:debug("Skipping ref for now: ~p ~p", [Ref, ExpandedRef]) - end, - ignore. - -base_attributes(Field, IOpts, FOpts) -> - Name = field_name(Field, FOpts), - {Name, [ - {id, field_id(Field, FOpts)}, - {name, Name}, - {required, maps:get(required, IOpts, false)} - ]}. - -render_input(select, Field, _, IOpts, FOpts) -> - {Name, Attributes} = base_attributes(Field, IOpts, FOpts), - OptionsHtml = lists:map(fun (Opt) -> {option, [], Opt} end, maps:get(values, IOpts, [])), - {'div', [{class, <<"json-field">>}], [ - {label, [{for, Name}], maps:get(label, IOpts, Field)}, - {select, Attributes, OptionsHtml} - ]}; - -render_input(input, Field, Type, IOpts, FOpts) -> - {Name, Attributes0} = base_attributes(Field, IOpts, FOpts), - Attributes = [ - {value, maps:get(value, IOpts, undefined)}, - {placeholder, maps:get(placeholder, IOpts, undefined)}, - {readonly, maps:get(readonly, IOpts, false)}, - {autocomplete, maps:get(autocomplete, IOpts, <<"off">>)}, - {type, maps:get(input_type, IOpts)}, - {class, <<"shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md">>} - | Attributes0], - HtmlNode = {'div', [{class, <<"json-field">>}], [ - {label, [{for, Name}, {class, <<"block text-sm font-medium text-gray-700">>}], maps:get(label, IOpts, Field)}, - {'div', [{class, <<"mt-1">>}], [{input, Attributes}]} - ]}, - HtmlNode. - -as_binary(Atom) when is_atom(Atom) -> - atom_to_binary(Atom); -as_binary(B) when is_binary(B) -> - B. - -input_type(_) -> <<"text">>. - -field_name(Field, Opts) -> - FB = as_binary(Field), - BaseName = maps:get(name, Opts, <<"form">>), - <<"[", BaseName/binary, "]", FB/binary>>. - -field_id(Field, Opts) -> - FB = as_binary(Field), - BaseId = maps:get(id, Opts, <<"form">>), - <>. diff --git a/apps/dreki_web/src/dreki_web_ui_node.erl b/apps/dreki_web/src/dreki_web_ui_node.erl deleted file mode 100644 index fdb7c77..0000000 --- a/apps/dreki_web/src/dreki_web_ui_node.erl +++ /dev/null @@ -1,23 +0,0 @@ --module(dreki_web_ui_node). --behaviour(cowboy_handler). --behaviour(cowboy_rest). --export([init/2]). --export([allowed_methods/2]). --export([content_types_accepted/2]). --export([content_types_provided/2]). - -init(Req, State) -> - {cowboy_rest, Req, State}. - -allowed_methods(Req, State) -> - {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. - -content_types_accepted(Req, State) -> - dreki_web:content_types_accepted(Req, State). - -content_types_provided(Req, State) -> - dreki_web:content_types_provided(Req, State). - -to_html(Req, State) -> - Html = dreki_web_ui:render(Req, node, []), - {Html, Req, State}. diff --git a/apps/dreki_web/src/dreki_web_ui_stores.erl b/apps/dreki_web/src/dreki_web_ui_stores.erl deleted file mode 100644 index d39e571..0000000 --- a/apps/dreki_web/src/dreki_web_ui_stores.erl +++ /dev/null @@ -1,126 +0,0 @@ --module(dreki_web_ui_stores). --behaviour(cowboy_handler). --export([init/2]). - - -init(Req, action) -> - with_location(Req#{action => cowboy_req:binding(action, Req)}, cowboy_req:binding(location, Req, undefined)); -init(Req, Action) -> - with_location(Req#{action => Action}, cowboy_req:binding(location, Req, undefined)). - -with_location(Req, undefined) -> - with_namespace(Req#{urn => dreki_world:root_path()}); -with_location(Req, <<"-">>) -> - with_namespace(Req#{urn => dreki_world:root_path()}); -with_location(Req0, Location) -> - Root = dreki_world:root_path(), - {ok, XUrn} = dreki_urn:expand(<>), - Req = Req0#{urn => maps:get(urn, XUrn)}, - with_namespace(Req). - -with_namespace(Req) -> - with_namespace(Req, cowboy_req:binding(namespace, Req)). -with_namespace(Req, undefined) -> - request(Req); -with_namespace(Req = #{urn := Urn0}, Namespace) -> - Urn = <>, - with_directory(Req#{urn => Urn}). - -with_directory(Req) -> - with_directory(Req, cowboy_req:binding(directory, Req)). -with_directory(Req, undefined) -> - request(Req); -with_directory(Req = #{urn := Urn0}, Directory) -> - Urn = <>, - with_id(Req#{urn => Urn}). - -with_id(Req) -> - with_id(Req, cowboy_req:binding(id, Req)). -with_id(Req, undefined) -> - request(Req); -with_id(Req = #{urn := Urn0}, Id) -> - Urn = <>, - request(Req#{urn => Urn}). - -request(Req = #{action := Action, method := Method, urn := Urn}) -> - logger:debug("Faked Urn ~p", [Urn]), - case dreki_urn:expand(Urn) of - {ok, XUrn} -> request(Method, Action, XUrn, Req); - {error, EMap=#{}} -> dreki_web_error:init(Req, EMap); - {error, _Error} -> dreki_web_error:init(Req, #{code => 404, status => "Not Found"}) - end. - -%% Stores list -request(<<"GET">>, undefined, #{resource := #{namespace := NS}}, Req) -> - {ok, Stores0} = dreki_store:stores(NS), - Stores = lists:map(fun (Store) -> - S = dreki_store:store_as_map(Store), - S#{href => urn_to_path(maps:get(urn, S))} - end, Stores0), - Html = dreki_web_ui:render(Req, namespace_dtl, [{namespace, NS}, {stores, Stores}]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; - -%% List -request(<<"GET">>, undefined, Urn = #{location := Loc, resource := #{directory := #{directory := Dir, namespace := NS}}}, Req) -> - {ok, Result} = dreki_store:list(Urn), - Results = lists:map(fun(Result) -> - Result#{href => urn_to_path(maps:get('@id', Result))} - end, maps:get(data, Result)), - Html = dreki_web_ui:render(Req, store_list_dtl, [ - {location, Loc}, {namespace, NS}, {directory, Dir}, {results, Results}, - {new, href(Req, <<"_/new">>)} - ]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; - -%% New -request(<<"GET">>, <<"new">>, #{urn := Urn, location := Loc, resource := #{directory := #{directory := Dir, namespace := NS}}}, Req) -> - logger:debug("Actual Urn: ~p", [Urn]), - {ok, Schemas} = dreki_store:list(<>), - {ok, Schema} = dreki_store:get(<>), - Form = dreki_web_ui_json_form:render_html(Schema, #{}), - Html = dreki_web_ui:render(Req, store_new_dtl, [ - {location, Loc}, {namespace, NS}, {directory, Dir}, - {schema, Schema}, {schemas, Schemas}, - {target, urn_to_path(Urn)}, {method, <<"POST">>}, - {form, Form} - ]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; - -%% Show -request(<<"GET">>, undefined, Urn = #{location := Loc, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}, Req) -> - {ok, Result0} = dreki_store:get(Urn), - Result = Result0#{'@href' => urn_to_path(Urn)}, - Html = dreki_web_ui:render(Req, store_show_dtl, [{location, Loc}, {id, Id}, {directory, Dir}, {namespace, NS}, {result, Result}]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}. - -derpinit(Req, _) -> - Json = #{<<"error">> => false, <<"service">> => <<"dreki">>}, - logger:debug("REQ: ~p", [Req]), - {ok, dreki_web:reply_json(Req, 200, Json), undefined}. - - -bad_request(Req) -> - dreki_web_error:init(Req, #{code => 400, status => "Bad request"}). - -href(Req = #{path := Path}, Append) -> - <>. - -location_to_path(Location) -> - Root = dreki_world:root_path(), - case Location =:= Root of - true -> <<"/admin/-">>; - false -> binary:replace(Location, <>, <<"/admin/">>) - end. - -urn_to_path(Urn) when is_binary(Urn) -> - {ok, XUrn} = dreki_urn:expand(Urn), - urn_to_path(XUrn); -urn_to_path(#{location := Location, resource := #{namespace := NS}}) -> - LP = location_to_path(Location), - <>; -urn_to_path(#{location := Location, resource := #{directory := #{directory := Dir, namespace := NS}}}) -> - LP = location_to_path(Location), - <>; -urn_to_path(#{location := Location, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}) -> - LP = location_to_path(Location), - <>. diff --git a/apps/dreki_web/src/dreki_web_ui_task.erl b/apps/dreki_web/src/dreki_web_ui_task.erl deleted file mode 100644 index 1ced583..0000000 --- a/apps/dreki_web/src/dreki_web_ui_task.erl +++ /dev/null @@ -1,20 +0,0 @@ --module(dreki_web_ui_task). --behaviour(cowboy_handler). --export([init/2]). - -init(Req = #{method := <<"GET">>}, State) -> - Id = cowboy_req:binding(id, Req), - {ok, Db} = dreki_tasks:open(), - case dreki_tasks:get(Db, Id) of - {ok, Task} -> - MTask = dreki_task:to_map(Task), - PrettyParams = jsone:encode(maps:get(params, MTask), [canonical_form, {space, 1}, {indent, 4}]), - Html = dreki_web_ui:render(Req, task_dtl, [{"page_title", <<"Task: ", Id/binary>>}, {"task", MTask}, {"task_pretty_params", PrettyParams}]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; - Error -> - logger:debug("Failed to lookup task ~p: ~p", [Id, Error]), - dreki_web_ui_error:init(Req, #{code => 404, status => "Not found"}) - end; -init(Req, _) -> - dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). - diff --git a/apps/dreki_web/src/dreki_web_ui_tasks.erl b/apps/dreki_web/src/dreki_web_ui_tasks.erl deleted file mode 100644 index e9748b8..0000000 --- a/apps/dreki_web/src/dreki_web_ui_tasks.erl +++ /dev/null @@ -1,14 +0,0 @@ --module(dreki_web_ui_tasks). --behaviour(cowboy_handler). --export([init/2]). - -init(Req = #{method := <<"GET">>}, State) -> - Local = maps:fold(fun - (Ln, #{mod := Mod, path := Path}, Acc) -> - [#{name => Ln, mod => Mod, path => Path, url => <<"/api/admin/tasks/", Path>>} | Acc] - end, [], dreki_tasks:local_stores()), - Html = dreki_web_ui:render(Req, tasks_dtl, [{"page_title", "Tasks"}, {"stores", Local}, {"tasks", []}]), - {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; -init(Req, _) -> - dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). - diff --git a/apps/dreki_web/src/ui/dreki_web_ui.erl b/apps/dreki_web/src/ui/dreki_web_ui.erl new file mode 100644 index 0000000..e394632 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui.erl @@ -0,0 +1,44 @@ +-module(dreki_web_ui). +-export([render/3, reply_html/3, reply_html/4]). + +render(Req, InnerModule, Assigns0) -> + Assigns = assigns(Req, Assigns0), + {ok, InnerHtml} = InnerModule:render(Assigns), + render_layout(Req, InnerHtml, Assigns). + +reply_html(Req, Code, Html) -> + reply_html(Req, Code, Html, #{}). + +reply_html(Req, Code, Html, Headers0) -> + Headers = maps:put(<<"content-type">>, <<"text/html">>, Headers0), + cowboy_req:reply(Code, Headers, Html, Req). + +render_layout(Req, InnerHtml, Assigns) -> + {ok, Html} = layout_dtl:render([{"inner", InnerHtml} | Assigns]), + Html. + +assigns(Req, Assigns0) -> + Assigns = clean_assigns(Assigns0), + [{"site_title", "Dreki"}, + {"identity_id", maps:get(identity_id, Req)}, + {"identity", maps:get(identity, Req)}, + {"identity_name", dreki_web:identity_name(Req)}, + {"dreki_node", node()}, + {"dreki_world", dreki_world:to_map()}, + {"trace_id", maps:get(<<"trace-id">>, Req, <<"no-trace">>)} + | Assigns]. + +content_types_accepted(Req, State) -> + {[ + {{ <<"multipart">>, <<"form-data">>, '*'}, from_form} + ], Req, State}. + +content_types_provided(Req, State) -> + {[{{ <<"text">>, <<"html">>, '*'}, to_html}], Req, State}. + +clean_assigns(Assigns) when is_list(Assigns) -> + Assigns; +clean_assigns(Map) when is_map(Map) -> + maps:fold(fun (Key, Value, Acc) -> [{Key, clean_assigns(Value)} | Acc] end, [], Map); +clean_assigns(Other) -> + Other. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_error.erl b/apps/dreki_web/src/ui/dreki_web_ui_error.erl new file mode 100644 index 0000000..ccda150 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_error.erl @@ -0,0 +1,14 @@ +-module(dreki_web_ui_error). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req, not_found) -> + reply(Req, 404, <<"Not Found">>, undefined); +init(Req, State = #{code := Code, status := Status}) -> + reply(Req, Code, Status, maps:get(message, State, undefined)). + +reply(Req0, Code, Status, Msg) -> + Assigns = [{"message", Msg}, {"status", Status}], + Html = dreki_web_ui:render(Req0, error_dtl, Assigns), + Req = dreki_web_ui:reply_html(Req0, Code, Html), + {ok, Req, undefined}. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_index.erl b/apps/dreki_web/src/ui/dreki_web_ui_index.erl new file mode 100644 index 0000000..9f4684e --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_index.erl @@ -0,0 +1,31 @@ +-module(dreki_web_ui_index). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req = #{method := <<"GET">>}, State) -> + PrettyWorld = jsone:encode(dreki_world:to_map(), [canonical_form, {space, 1}, {indent, 4}]), + LocalTasksStores = dreki_tasks:local_stores(), + {ok, Peers} = partisan_peer_service:members(), + NavTree = nav_tree(dreki_world_dns:as_map()), + Html = dreki_web_ui:render(Req, index_dtl, [ + {"page_title", "Admin UI"}, + {"dreki_world_pretty_json", PrettyWorld}, + {"dreki_local_tasks_stores", LocalTasksStores}, + {"dreki_peers", Peers}, + {"nav_tree", NavTree} + ]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; +init(Req, _) -> + dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). + +nav_tree(#{vertices := Vertices}) -> + nav_tree(Vertices, []). + +nav_tree([Region = #{type := region, name := Name, data := #{display_name := Display}} | Rest], Acc) -> + nav_tree(Rest, [Region#{href => <<"/admin/regions/", Name/binary>>, title => Display} | Acc]); +nav_tree([Node = #{type := node, name := Name, data := #{display_name := Display}} | Rest], Acc) -> + nav_tree(Rest, [Node#{href => <<"/admin/nodes/", Name/binary>>, title => Display} | Acc]); +nav_tree([#{type := Root} | Rest], Acc) -> + nav_tree(Rest, Acc); +nav_tree([], Acc) -> + Acc. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_json_form.erl b/apps/dreki_web/src/ui/dreki_web_ui_json_form.erl new file mode 100644 index 0000000..51a9de4 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_json_form.erl @@ -0,0 +1,160 @@ +-module(dreki_web_ui_json_form). +-export([render_html/2]). +-export([render/2]). +-export([to_html/1]). + +-type dreki_form() :: #{ + input => binary(), + label => binary() +}. + +render_html(Schema, Opts) -> + Abstract = render(Schema, Opts), + to_html(Abstract). + +to_html(Atom) when is_atom(Atom) -> + as_binary(Atom); +to_html(Binary) when is_binary(Binary) -> + Binary; +to_html(Abstract) -> + to_html(Abstract, []). + +to_html([{Node, Attrs} | Rest], Acc) -> + BNode = as_binary(Node), + AttrsS = attrs_to_html(Attrs), + to_html(Rest, [[<<"<">>, BNode, <<" ">>, AttrsS, <<">">>] | Acc]); +to_html([{Node, Attrs, Content} | Rest], Acc) -> + BNode = as_binary(Node), + AttrsS = attrs_to_html(Attrs), + logger:debug("Node ~p Attrs ~p Content ~p", [Node, Attrs, Content]), + to_html(Rest, [[<<"<">>, BNode, <<" ">>, AttrsS, <<">">>, to_html(Content), <<">, BNode, <<">">>] | Acc]); +to_html([List | Rest], Acc) when is_list(List) -> + to_html(Rest, [to_html(List) | Acc]); +to_html([], Acc) -> + lists:reverse(Acc). + +attrs_to_html(Attrs) -> attrs_to_html(Attrs, []). +attrs_to_html([{Attr, undefined} | Rest], Acc) -> + attrs_to_html(Rest, Acc); +attrs_to_html([{Attr, false} | Rest], Acc) -> + attrs_to_html(Rest, Acc); +attrs_to_html([{Attr, true} | Rest], Acc) -> + attrs_to_html(Rest, [[as_binary(Attr), <<" ">>] | Acc]); +attrs_to_html([{Attr, Num} | Rest], Acc) when is_integer(Num) -> + attrs_to_html(Rest, [[as_binary(Attr), <<"=">>, Num, <<" ">>] | Acc]); +attrs_to_html([{Attr, Value} | Rest], Acc) -> + attrs_to_html(Rest, [[as_binary(Attr), <<"=">>, <<"\"">>, as_binary(Value), <<"\"">>, <<" ">>] | Acc]); +attrs_to_html([], Acc) -> + Acc. + +put_new(Key, Value, Map) -> + case maps:is_key(Key, Map) of + true -> + Map; + false -> + maps:put(Key, Value, Map) + end. + +render(Schema, Opts) -> + render(Schema, Opts, {'div', [{class, <<"mt-6 grid grid-cols-1 gap-y-6 gap-x-4 sm:grid-cols-6">>}]}). + +render(Schema, Opts, {Elem, Attrs}) -> + [{Elem, Attrs, render(Schema, Opts, undefined)}]; +render(Schema, Opts0, undefined) -> + Opts = put_new(name, <<"form">>, Opts0), + lists:reverse(maps:fold(fun (Key, Value, Acc) -> + case render_property(Key, Value, Schema, Schema, Opts) of + {ok, Html} -> [Html | Acc]; + ignore -> Acc + end + end, [], maps:get(properties, Schema))). + +render_property(Field, Config = #{enum := Enum}, Parent, Schema, Opts) -> + FormAttrs = maps:get(<<"dreki:form">>, Config, #{}), + Required = lists:member(Field, maps:get(required, Parent, [])), + Label = maps:get(label, FormAttrs, maps:get(title, Config, Field)), + InputOpts = #{required => Required, label => Label, values => Enum}, + {ok, render_input(select, Field, undefined, InputOpts, Opts)}; + +render_property(Field, Config = #{type := Type}, Parent, Schema, Opts) -> + FormAttrs = maps:get(<<"dreki:form">>, Config, #{}), + Input = maps:get(input, FormAttrs, input), + InputType = maps:get(input_type, FormAttrs, input_type(Type)), + Required = lists:member(Field, maps:get(required, Parent, [])), + Label = maps:get(label, FormAttrs, maps:get(title, Config, Field)), + InputOpts = #{required => Required, label => Label, input_type => InputType}, + {ok, render_input(Input, Field, Type, InputOpts, Opts)}; + +render_property(Key, SubSchema = #{<<"@schema">> := SubSchemaUrn}, Parent, Schema, Opts) -> + Name = maps:get(name, Opts), + KeyB = atom_to_binary(Key), + InnerForm = render(SubSchema, Opts#{name => <>}, undefined), + Head = {'div', [{class, <<"sm:col-span-6 mb-3">>}], [ + {h3, [{class, <<"text-lg leading-6 font-medium text-gray-900 dark:text-gray-200">>}], maps:get(title, SubSchema, SubSchemaUrn)}, + {p, [{class, <<"mt-1 text-sm text-gray-500 dark:text-gray-400">>}], maps:get(description, SubSchema, <<>>)} + ]}, + {ok, {'div', [{class, <<"sm:col-span-6">>}], + [ + {'div', [{class, <<"mt-6 grid grid-cols-1 gap-y-6 gap-x-4 sm:grid-cols-6">>}], [Head, InnerForm]} + ]}}; + +render_property(Field, #{'$ref' := Ref}, Parent, Schema = #{'$defs' := Defs}, Opts) -> + case maps:get(Ref, Defs, undefined) of + undefined -> logger:error("didn't get ref ~p", [Ref]); + ExpandedRef -> + logger:debug("Skipping ref for now: ~p ~p", [Ref, ExpandedRef]) + end, + ignore. + +base_attributes(Field, IOpts, FOpts) -> + Name = field_name(Field, FOpts), + {Name, [ + {id, field_id(Field, FOpts)}, + {name, Name}, + {required, maps:get(required, IOpts, false)} + ]}. + +render_input(select, Field, _, IOpts, FOpts) -> + {Name, Attributes} = base_attributes(Field, IOpts, FOpts), + OptionsHtml = lists:map(fun (Opt) -> {option, [], Opt} end, maps:get(values, IOpts, [])), + {'div', [{class, <<"json-field">>}], + [ + label(Field, Name, maps:get(label, IOpts, Field)), + {select, Attributes, OptionsHtml} + ]}; + +render_input(input, Field, Type, IOpts, FOpts) -> + {Name, Attributes0} = base_attributes(Field, IOpts, FOpts), + Attributes = [ + {value, maps:get(value, IOpts, undefined)}, + {placeholder, maps:get(placeholder, IOpts, undefined)}, + {readonly, maps:get(readonly, IOpts, false)}, + {autocomplete, maps:get(autocomplete, IOpts, <<"off">>)}, + {type, maps:get(input_type, IOpts)}, + {class, <<"shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md dark:bg-gray-800">>} + | Attributes0], + {'div', [{class, <<"json-field sm:col-span-6">>}], + [ + label(Field, Name, maps:get(label, IOpts, Field)), + {'div', [{class, <<"mt-1">>}], [{input, Attributes}]} + ]}. + +label(_Field, Name, Label) -> + {label, [{for, Name}, {class, <<"block text-sm font-medium dark:text-gray-400 text-gray-700 dark:text-gray-400">>}], Label}. + +as_binary(Atom) when is_atom(Atom) -> + atom_to_binary(Atom); +as_binary(B) when is_binary(B) -> + B. + +input_type(_) -> <<"text">>. + +field_name(Field, Opts) -> + FB = as_binary(Field), + BaseName = maps:get(name, Opts, <<"form">>), + <>. + +field_id(Field, Opts) -> + FB = as_binary(Field), + BaseId = maps:get(id, Opts, <<"form">>), + <>. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_node.erl b/apps/dreki_web/src/ui/dreki_web_ui_node.erl new file mode 100644 index 0000000..fdb7c77 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_node.erl @@ -0,0 +1,23 @@ +-module(dreki_web_ui_node). +-behaviour(cowboy_handler). +-behaviour(cowboy_rest). +-export([init/2]). +-export([allowed_methods/2]). +-export([content_types_accepted/2]). +-export([content_types_provided/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +allowed_methods(Req, State) -> + {[<<"GET">>, <<"HEAD">>, <<"OPTIONS">>, <<"POST">>], Req, State}. + +content_types_accepted(Req, State) -> + dreki_web:content_types_accepted(Req, State). + +content_types_provided(Req, State) -> + dreki_web:content_types_provided(Req, State). + +to_html(Req, State) -> + Html = dreki_web_ui:render(Req, node, []), + {Html, Req, State}. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_stores.erl b/apps/dreki_web/src/ui/dreki_web_ui_stores.erl new file mode 100644 index 0000000..e39cd48 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_stores.erl @@ -0,0 +1,189 @@ +-module(dreki_web_ui_stores). +-behaviour(cowboy_handler). +-export([init/2]). + + +init(Req, action) -> + with_location(Req#{action => cowboy_req:binding(action, Req)}, cowboy_req:binding(location, Req, undefined)); +init(Req, Action) -> + with_location(Req#{action => Action}, cowboy_req:binding(location, Req, undefined)). + +with_location(Req, undefined) -> + with_namespace(Req#{urn => dreki_world:root_path()}); +with_location(Req, <<"-">>) -> + with_namespace(Req#{urn => dreki_world:root_path()}); +with_location(Req0, Location) -> + Root = dreki_world:root_path(), + {ok, XUrn} = dreki_urn:expand(<>), + Req = Req0#{urn => maps:get(urn, XUrn)}, + with_namespace(Req). + +with_namespace(Req) -> + with_namespace(Req, cowboy_req:binding(namespace, Req)). +with_namespace(Req, undefined) -> + request(Req); +with_namespace(Req = #{urn := Urn0}, Namespace) -> + Urn = <>, + with_directory(Req#{urn => Urn}). + +with_directory(Req) -> + with_directory(Req, cowboy_req:binding(directory, Req)). +with_directory(Req, undefined) -> + request(Req); +with_directory(Req = #{urn := Urn0}, Directory) -> + Urn = <>, + with_id(Req#{urn => Urn}). + +with_id(Req) -> + with_id(Req, cowboy_req:binding(id, Req)). +with_id(Req, undefined) -> + request(Req); +with_id(Req = #{urn := Urn0}, Id) -> + Urn = <>, + request(Req#{urn => Urn}). + +request(Req = #{action := Action, method := Method, urn := Urn}) -> + logger:debug("Faked Urn ~p", [Urn]), + case dreki_urn:expand(Urn) of + {ok, XUrn} -> request(Method, Action, XUrn, Req); + {error, EMap=#{}} -> dreki_web_error:init(Req, EMap); + {error, _Error} -> dreki_web_error:init(Req, #{code => 404, status => "Not Found"}) + end. + +%% Stores list +request(<<"GET">>, undefined, #{resource := #{namespace := NS}}, Req) -> + {ok, Stores0} = dreki_store:stores(NS), + Stores = lists:map(fun (Store) -> + S = dreki_store:store_as_map(Store), + S#{href => urn_to_path(maps:get(urn, S))} + end, Stores0), + Html = dreki_web_ui:render(Req, namespace_dtl, [{namespace, NS}, {stores, Stores}]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; + +%% List +request(<<"GET">>, undefined, Urn = #{location := Loc, resource := #{directory := #{directory := Dir, namespace := NS}}}, Req) -> + {ok, Result} = dreki_store:list(Urn), + Results0 = lists:map(fun(Result) -> + Result#{href => urn_to_path(maps:get('@id', Result))} + end, maps:get(data, Result)), + Results = deep_map_to_list(Results0), + Html = dreki_web_ui:render(Req, store_list_dtl, [ + {location, Loc}, {namespace, NS}, {directory, Dir}, + {results, Results}, + {actions, [ + [{title, "New"}, {href, href(Req, <<"_/new">>)}] + ]} + ]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; + +%% New +request(<<"GET">>, <<"new">>, #{urn := Urn, location := Loc, resource := #{directory := #{directory := Dir, namespace := NS}}}, Req) -> + logger:debug("Actual Urn: ~p", [Urn]), + {ok, Schemas} = dreki_store:list(<>), + {ok, Schema} = dreki_store:get(<>), + Form = dreki_web_ui_json_form:render_html(Schema, #{}), + Html = dreki_web_ui:render(Req, store_new_dtl, [ + {location, Loc}, {namespace, NS}, {directory, Dir}, + {schema, Schema}, {schemas, Schemas}, + {target, urn_to_path(Urn)}, {method, <<"POST">>}, + {form, Form} + ]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; + +%% Show +request(<<"GET">>, undefined, Urn = #{location := Loc, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}, Req) -> + {ok, Result0} = dreki_store:get(Urn), + Result = Result0#{'@href' => urn_to_path(Urn)}, + Actions = lists:foldr(fun (#{id := Id, title := Title}, Acc) -> + IdB = atom_to_binary(Id), + Action = [{id, Id}, {title, Title}, {href, href(Req, <<"_/", IdB/binary>>)}], + [Action | Acc] + end, [], maps:get('@actions', Result)), + Html = dreki_web_ui:render(Req, store_show_dtl, [{location, Loc}, {id, Id}, {directory, Dir}, {namespace, NS}, + {result, deep_map_to_list(Result)}, + {actions, [ + [{title, "Edit (UI)"}, {href, href(Req, <<"_/edit">>)}] + | Actions + ]} + ]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), undefined}; + +%% Show action +request(<<"GET">>, Action0, Urn = #{location := Loc, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}, Req) -> + {ok, Result0} = dreki_store:get(Urn), + Result = Result0#{'@href' => urn_to_path(Urn)}, + Action = binary_to_existing_atom(Action0), + Path = href(Req, <<"_/", Action0/binary>>), + Act = lists:filter(fun (A) -> maps:get(id, A) =:= Action end, maps:get('@actions', Result)), + case Act of + [#{title := Title, new := {Mod, Fun, Args}}] -> + {ok, Schema} = apply(Mod, Fun, [Result | Args]), + Form = dreki_web_ui_json_form:render_html(Schema, #{}), + Html = dreki_web_ui:render(Req, store_new_dtl, [ + {location, Loc}, {id, Id}, {directory, Dir}, {namespace, NS}, + {method, <<"POST">>}, + {title, Title}, + {result, deep_map_to_list(Result)}, + {target, Path}, + {form, Form} + ]), + {dreki_web_ui:reply_html(Req, 200, Html), undefined}; + _ -> + dreki_web_ui_error:init(Req, #{code => 404, status => "Not Found", message => "No such action"}) + end; +request(<<"POST">>, Action0, Urn = #{location := Loc, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}, Req) -> + {ok, Result0} = dreki_store:get(Urn), + Result = Result0#{'@href' => urn_to_path(Urn)}, + Action = binary_to_existing_atom(Action0), + Act = lists:filter(fun (A) -> maps:get(id, A) =:= Action end, maps:get('@actions', Result)), + case Act of + [#{title := Title, new := {Mod, Fun, Args}}] -> + Req; + _ -> + dreki_web_ui_error:init(Req, #{code => 404, status => "Not Found", message => "No such action"}) + end. + +derpinit(Req, _) -> + Json = #{<<"error">> => false, <<"service">> => <<"dreki">>}, + logger:debug("REQ: ~p", [Req]), + {ok, dreki_web:reply_json(Req, 200, Json), undefined}. + + +bad_request(Req) -> + dreki_web_error:init(Req, #{code => 400, status => "Bad request"}). + +href(Req = #{path := Path}, Append) -> + <>. + +location_to_path(Location) -> + Root = dreki_world:root_path(), + case Location =:= Root of + true -> <<"/admin/-">>; + false -> binary:replace(Location, <>, <<"/admin/">>) + end. + +deep_map_to_list(List) when is_list(List) -> + [deep_map_to_list(Map) || Map <- List]; +deep_map_to_list(Map) when is_map(Map) -> + deep_map_to_list(maps:to_list(Map), []). + +deep_map_to_list([{Key, Map} | Rest], Acc) when is_map(Map) -> + List = deep_map_to_list(Map), + deep_map_to_list(Rest, [{Key, List} | Acc]); +deep_map_to_list([Item | Rest], Acc) -> + deep_map_to_list(Rest, [Item | Acc]); +deep_map_to_list([], Acc) -> + Acc. + +urn_to_path(Urn) when is_binary(Urn) -> + {ok, XUrn} = dreki_urn:expand(Urn), + urn_to_path(XUrn); +urn_to_path(#{location := Location, resource := #{namespace := NS}}) -> + LP = location_to_path(Location), + <>; +urn_to_path(#{location := Location, resource := #{directory := #{directory := Dir, namespace := NS}}}) -> + LP = location_to_path(Location), + <>; +urn_to_path(#{location := Location, resource := #{resource := #{id := Id, directory := Dir, namespace := NS}}}) -> + LP = location_to_path(Location), + <>. diff --git a/apps/dreki_web/src/ui/dreki_web_ui_task.erl b/apps/dreki_web/src/ui/dreki_web_ui_task.erl new file mode 100644 index 0000000..1ced583 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_task.erl @@ -0,0 +1,20 @@ +-module(dreki_web_ui_task). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req = #{method := <<"GET">>}, State) -> + Id = cowboy_req:binding(id, Req), + {ok, Db} = dreki_tasks:open(), + case dreki_tasks:get(Db, Id) of + {ok, Task} -> + MTask = dreki_task:to_map(Task), + PrettyParams = jsone:encode(maps:get(params, MTask), [canonical_form, {space, 1}, {indent, 4}]), + Html = dreki_web_ui:render(Req, task_dtl, [{"page_title", <<"Task: ", Id/binary>>}, {"task", MTask}, {"task_pretty_params", PrettyParams}]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; + Error -> + logger:debug("Failed to lookup task ~p: ~p", [Id, Error]), + dreki_web_ui_error:init(Req, #{code => 404, status => "Not found"}) + end; +init(Req, _) -> + dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). + diff --git a/apps/dreki_web/src/ui/dreki_web_ui_tasks.erl b/apps/dreki_web/src/ui/dreki_web_ui_tasks.erl new file mode 100644 index 0000000..e9748b8 --- /dev/null +++ b/apps/dreki_web/src/ui/dreki_web_ui_tasks.erl @@ -0,0 +1,14 @@ +-module(dreki_web_ui_tasks). +-behaviour(cowboy_handler). +-export([init/2]). + +init(Req = #{method := <<"GET">>}, State) -> + Local = maps:fold(fun + (Ln, #{mod := Mod, path := Path}, Acc) -> + [#{name => Ln, mod => Mod, path => Path, url => <<"/api/admin/tasks/", Path>>} | Acc] + end, [], dreki_tasks:local_stores()), + Html = dreki_web_ui:render(Req, tasks_dtl, [{"page_title", "Tasks"}, {"stores", Local}, {"tasks", []}]), + {ok, dreki_web_ui:reply_html(Req, 200, Html), State}; +init(Req, _) -> + dreki_web_ui_error:init(Req, #{code => 400, status => "Bad request"}). + diff --git a/apps/dreki_web/templates/crash.dtl b/apps/dreki_web/templates/crash.dtl new file mode 100644 index 0000000..b0a291c --- /dev/null +++ b/apps/dreki_web/templates/crash.dtl @@ -0,0 +1,52 @@ + + + + + + {% if page_title %}{{ page_title }} - {% endif %}{{site_title}} + + + + + + + +
+
+ + +
+
+ +
+
+

😤
{{ class }}

+

{{ reason }}

+ {% for line in stacktrace %} +

{{ line }}

+ {% endfor %} +
+
+ + + + + diff --git a/apps/dreki_web/templates/layout.dtl b/apps/dreki_web/templates/layout.dtl index b27fe30..a326305 100644 --- a/apps/dreki_web/templates/layout.dtl +++ b/apps/dreki_web/templates/layout.dtl @@ -73,10 +73,11 @@

- dreki $nodename + dreki {{dreki_node}}

-

- identity:{{identity_id}} +

+ identity: {{identity_id}}
+ trace: {{trace_id}}>

diff --git a/apps/dreki_web/templates/store_list.dtl b/apps/dreki_web/templates/store_list.dtl index 65dc5af..81209a6 100644 --- a/apps/dreki_web/templates/store_list.dtl +++ b/apps/dreki_web/templates/store_list.dtl @@ -1,9 +1,11 @@

{{ location }} :: {{ namespace }}:{{ directory }}

 -Create +{% for action in actions %} +{{ action.title }} +{% endfor %} diff --git a/apps/dreki_web/templates/store_new.dtl b/apps/dreki_web/templates/store_new.dtl index 5a85854..0ae8d59 100644 --- a/apps/dreki_web/templates/store_new.dtl +++ b/apps/dreki_web/templates/store_new.dtl @@ -2,14 +2,18 @@
{{ location }} / {{ namespace }} / {{ directory }}
- New + {% if result %} + {{ result.id }}: {{ title }} + {% else %} + New + {% endif %} -
+ {{ form | safe }}
-
+
diff --git a/apps/dreki_web/templates/store_show.dtl b/apps/dreki_web/templates/store_show.dtl index e944bee..3b139bb 100644 --- a/apps/dreki_web/templates/store_show.dtl +++ b/apps/dreki_web/templates/store_show.dtl @@ -5,4 +5,12 @@ {{ result.id }} +
    +{% for res in result %} +
  • {{res}}
    • +{% endfor %} +
+{% for action in actions %} +{{ action.title }} +{% endfor %} diff --git a/config/mgmt2/sys.config b/config/mgmt2/sys.config new file mode 100644 index 0000000..cb272d4 --- /dev/null +++ b/config/mgmt2/sys.config @@ -0,0 +1,114 @@ +[ +{kernel, + [ + {logger_level, debug}, + {logger, [ + {handler, default, logger_std_h, + #{level => debug, + formatter => {logger_colorful_formatter, + #{ + colors => #{debug => blue, notice => {bright, green}, warning => yellow, error => red, + critical => {bright, red}, alert => {bright, magenta}, emergency => {bg, red}}, + template => [ + time, " ", + "level=",level," ", + {trace_id, ["trace_id=", trace_id, " "], []}, + {domain, ["domain=", domain, " "], []}, + {mfa, ["mfa=", mfa, " "], []}, + {pid, ["pid=", pid, " "], []}, + msg, " ", + %%{file, ["file=", file, ":", line, " "], []}, + "\n" + ] + }}, + filters => [ + {opentelemetry_logger_metadata, {fun opentelemetry_logger_metadata:filter/2, []}} + ] + }} + ]} + ]}, + + +{opentelemetry, [ + {span_processor, batch}, + {exporter, otlp}, + {text_map_propagators, [baggage, trace_context]} +]}, + +{opentelemetry_exporter, [ + {otlp_protocol, grpc}, + {otlp_endpoint, "http://tempo.stairway.internal.random.sh:4317"}, + {otlp_headers, []} +]}, + + + +{plum_db, [ + {aae_enabled, true}, + {store_open_retries_delay, 2000}, + {store_open_retry_Limit, 30}, + {data_exchange_timeout, 60000}, + {hashtree_timer, 10000}, + {data_dir, "data2/plumdb"}, + {partitions, 8}, + {prefixes, [ + {state, ram}, + {world, ram_disk}, + {names, ram_disk}, + {regions, ram_disk}, + {nodes, ram_disk}, + {paths, ram_disk}, + {tasks, ram_disk}, + {objects, ram_disk}, + {stores, ram_disk}, + {'idx:roles', ram_disk}, {'idx:tags', ram_disk} + ]} +]}, +{partisan, [ + {peer_ip, {0,0,0,0}}, + {peer_port, 18087}, % port for inter-node communication + {parallelism, 4}, % number of tcp connections + {pid_encoding, false}, + {ref_encoding, false}, + {exchange_tick_period, 60000}, + {lazy_tick_period, 1000}, + {partisan_peer_service_manager, + partisan_pluggable_peer_service_manager} +]}, +{plumtree, [ + {broadcast_exchange_timer, 60000} % Perform AAE exchange every 1 min. +]}, + {dreki, [ + {root_domain, <<"random.sh">>}, + {internal_domain, <<"inf.random.sh">>}, + {domain, <<"mgmt2.stairway.dc2.scw.fr.eu.inf.random.sh">>}, + {local_tasks_stores, [ + {<<"local">>, dreki_dets_tasks, #{}, #{}} + ]}, + {autojoin, [<<"mgmt.stairway.dc2.scw.fr.eu.inf.random.sh">>]}, + {local_names_store, [ + %% Store for *.DOMAIN + {local, dreki_dets_names, []}, + %% Store for XXX.DOMAIN + {<<"service">>, dreki_dets_names, []} + ]}, + {local_sequences_store, {dreki_dets_sequences, []}} + ]}, + {ory, [ + {kratos_url, <<"https://kratos.sso.internal.random.sh">>}, + {hydra_url, <<"https://hydra.sso.internal.random.sh">>}, + {keto_url, <<"https://keto.sso.internal.random.sh">>}, + {hackney_ssl_opts, [ + {verify, verify_peer}, + {versions, ['tlsv1.2', 'tlsv1.3']}, + {cacertfile, "/usr/local/etc/ssl/certs/ca.internal.random.sh.crt"}, + {crl_check, false}, + {crl_cache, {ssl_crl_cache, {internal, [{http, 5000}]}}} + ]} + ]}, + {dreki_web, [ + {transport, [ + {port, 5001} + ]} + ]} +]. diff --git a/config/mgmt2/vm.args b/config/mgmt2/vm.args new file mode 100644 index 0000000..ea807ca --- /dev/null +++ b/config/mgmt2/vm.args @@ -0,0 +1,9 @@ +-name dreki2@mgmt2.stairway.dc2.scw.fr.eu.inf.random.sh + +-setcookie dreki_cookie + ++K true ++A30 + +## Enable multi_time_warp ++C multi_time_warp diff --git a/config/sys.config b/config/sys.config index be53a1f..5037a42 100644 --- a/config/sys.config +++ b/config/sys.config @@ -1,20 +1,32 @@ [ -{kernel, [ - {logger_level, info}, - {logger, [ - {handler, default, logger_std_h, #{ - formatter => - {logger_colorful_formatter, #{ - colors => #{debug => blue, notice => {bright, green}, warning => yellow, error => red, - critical => {bright, red}, alert => {bright, magenta}, emergency => {bg, red}}, - template => [time," [",level,"] ", file,":",line," ",msg,"\n"] - }} - }} -%%#{ formatter => -%% {logger_colorful_formatter, #{}} %%#{ template => [time," [",level,"] ", file,":",line," ",msg,"\n\n"] }} -%% }} - ]} -]}, +{kernel, + [ + {logger_level, debug}, + {logger, [ + {handler, default, logger_std_h, + #{level => info, + formatter => {logger_colorful_formatter, + #{ + colors => #{debug => blue, notice => {bright, green}, warning => yellow, error => red, + critical => {bright, red}, alert => {bright, magenta}, emergency => {bg, red}}, + template => [ + time, " ", + "level=",level," ", + {trace_id, ["trace_id=", trace_id, " "], []}, + {domain, ["domain=", domain, " "], []}, + {mfa, ["mfa=", mfa, " "], []}, + {pid, ["pid=", pid, " "], []}, + msg, " ", + %%{file, ["file=", file, ":", line, " "], []}, + "\n" + ] + }}, + filters => [ + {opentelemetry_logger_metadata, {fun opentelemetry_logger_metadata:filter/2, []}} + ] + }} + ]} + ]}, %%{lager, [ %% {error_logger_redirect, false}, @@ -23,18 +35,14 @@ {opentelemetry, [ {span_processor, batch}, - {exporter, {otel_exporter_stdout, []}}, + {exporter, otlp}, {text_map_propagators, [baggage, trace_context]} ]}, {opentelemetry_exporter, [ - {oltp_protocol, grpc}, - {otlp_endpoint, "https://tempo-us-central1.grafana.net:443"}, - {oltp_headers, [ - %% 38972:eyJrIjoiMzg1ZjEwZTg3YjU0ZDY4ZGQzZTg3MzllNzU4NGZlZjI1NmQ5YWRhMCIsIm4iOiJkcmVraSBkZXYiLCJpZCI6NDg1NjI5fQ== - {"authorization", - "Basic Mzg5NzI6ZXlKcklqb2lNemcxWmpFd1pUZzNZalUwWkRZNFpHUXpaVGczTXpsbE56VTROR1psWmpJMU5tUTVZV1JoTUNJc0ltNGlPaUprY21WcmFTQmtaWFlpTENKcFpDSTZORGcxTmpJNWZRPT0="} - ]} + {otlp_protocol, grpc}, + {otlp_endpoint, "http://tempo.stairway.internal.random.sh:4317"}, + {otlp_headers, []} ]}, {plum_db, [ diff --git a/rebar.config b/rebar.config index 05d8e4c..99d2e26 100644 --- a/rebar.config +++ b/rebar.config @@ -5,6 +5,15 @@ {opentelemetry_api, "~> 1.0"}, {opentelemetry, "~> 1.0"}, {opentelemetry_exporter, "1.0.2"}, + {opentelemetry_process_propagator, "0.1.1"}, + {opentelemetry_logger_metadata, "0.1.0"}, + {opentelemetry_telemetry, "~> 1.0.0-beta.7"}, + {telemetry, "~> 1.0"}, + {prometheus, "~> 4.8.2"}, + + {circuit_breaker, {git, "https://github.com/klarna/circuit_breaker", {branch, "master"}}}, + {sbroker, "1.0.0"}, + {genlib, {git, "https://github.com/rbkmoney/genlib", {branch, "master"}}}, {uuid, "2.0.4", {pkg, uuid_erl}}, {ory, {git, "https://git.random.sh/erlang-ory.git", {branch, "main"}}}, {mnesia_rocksdb, {git, "https://github.com/aeternity/mnesia_rocksdb", {branch, "master"}}}, @@ -30,7 +39,7 @@ rebar3_lint ]}. -%%{elvis_output_format, plain | colors | parsable}. +%%{elvis_output_format, plain | colors | parsable}. {elvis_output_format, parsable}. {elvis, [ #{ dirs => ["apps/*/src/**", "src/**"], diff --git a/rebar.lock b/rebar.lock index 2f0a5bf..c5d97ad 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,6 @@ {"1.2.0", -[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1}, +[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2}, + {<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1}, {<<"app_config">>, {git,"https://gitlab.com/leapsight/app_config.git", {ref,"e6a4dc99c0c9f17a6d4d865e40aefc409986f849"}}, @@ -14,6 +15,10 @@ {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},2}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.6.1">>},2}, {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.11.0">>},2}, + {<<"circuit_breaker">>, + {git,"https://github.com/klarna/circuit_breaker", + {ref,"ca94b561605e6319fabd45a2120c0cfd3edf6820"}}, + 0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0}, {<<"cowboy_telemetry">>,{pkg,<<"cowboy_telemetry">>,<<"0.4.0">>},0}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},1}, @@ -37,6 +42,10 @@ {ref,"dd6fb59a82ad0b920baee97271826b1ae255d1b4"}}, 0}, {<<"erlsom">>,{pkg,<<"erlsom">>,<<"1.5.0">>},2}, + {<<"erltrace">>, + {git,"https://gitlab.com/Project-FiFo/FiFo/erltrace.git", + {ref,"66d0c3c42284fb692525dd65f89cf1446b8787b0"}}, + 0}, {<<"erlydtl">>,{pkg,<<"erlydtl">>,<<"0.14.0">>},0}, {<<"fast_yaml">>, {git,"https://github.com/processone/fast_yaml", @@ -44,12 +53,17 @@ 0}, {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.8">>},1}, {<<"gen_batch_server">>,{pkg,<<"gen_batch_server">>,<<"0.8.7">>},1}, + {<<"genlib">>, + {git,"https://github.com/rbkmoney/genlib", + {ref,"82c5ff3866e3019eb347c7f1d8f1f847bed28c10"}}, + 0}, {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1}, {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.14.0">>},1}, {<<"hackney">>,{pkg,<<"hackney">>,<<"1.17.4">>},1}, {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2}, + {<<"inet_cidr">>,{pkg,<<"erl_cidr">>,<<"1.2.0">>},0}, {<<"iso8601">>,{pkg,<<"iso8601">>,<<"1.3.1">>},1}, {<<"jesse">>, {git,"https://github.com/for-GET/jesse", @@ -97,6 +111,12 @@ {<<"opentelemetry_exporter">>, {pkg,<<"opentelemetry_exporter">>,<<"1.0.2">>}, 0}, + {<<"opentelemetry_logger_metadata">>, + {pkg,<<"opentelemetry_logger_metadata">>,<<"0.1.0">>}, + 0}, + {<<"opentelemetry_process_propagator">>, + {pkg,<<"opentelemetry_process_propagator">>,<<"0.1.1">>}, + 0}, {<<"opentelemetry_telemetry">>, {pkg,<<"opentelemetry_telemetry">>,<<"1.0.0-beta.7">>}, 1}, @@ -114,6 +134,10 @@ {git,"https://gitlab.com/leapsight/plum_db", {ref,"49ab98faed623e26a2293eac0b73bed34b02e47a"}}, 0}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.2">>},0}, + {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.8">>},0}, + {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.11">>},1}, + {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1}, {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.4">>},1}, {<<"ra">>,{pkg,<<"ra">>,<<"2.0.6">>},0}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},1}, @@ -124,6 +148,7 @@ 1}, {<<"rfc3339">>,{pkg,<<"rfc3339">>,<<"0.2.2">>},1}, {<<"rocksdb">>,{pkg,<<"rocksdb">>,<<"1.7.0">>},1}, + {<<"sbroker">>,{pkg,<<"sbroker">>,<<"1.0.0">>},0}, {<<"sext">>,{pkg,<<"sext">>,<<"1.8.0">>},1}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},2}, {<<"telemetry">>,{pkg,<<"telemetry">>,<<"1.1.0">>},1}, @@ -142,6 +167,7 @@ {<<"yamerl">>,{pkg,<<"yamerl">>,<<"0.10.0">>},0}]}. [ {pkg_hash,[ + {<<"accept">>, <<"B33B127ABCA7CC948BBE6CAA4C263369ABF1347CFA9D8E699C6D214660F10CD1">>}, {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>}, {<<"aten">>, <<"F88EB38CEADC0710B84B5D78F0357D766733D80902D50AC69B70CDF834992DAA">>}, {<<"base32">>, <<"044F6DC95709727CA2176F3E97A41DDAA76B5BC690D3536908618C0CB32616A2">>}, @@ -165,6 +191,7 @@ {<<"hackney">>, <<"99DA4674592504D3FB0CFEF0DB84C3BA02B4508BAE2DFF8C0108BAA0D6E0977C">>}, {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, + {<<"inet_cidr">>, <<"9205FFB290C0DE8D2B82147976602FBF5BFA6D594834E60556AFAF3B82856B95">>}, {<<"iso8601">>, <<"D1CEE73F56D71C35590C6B2DB2074873BF410BABAAB768F6EA566366D8CA4810">>}, {<<"jsone">>, <<"7EA1098FE004C4127320FE0E3CF6A951B01F82039FEAA56C322DC7E34DD59762">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, @@ -178,15 +205,22 @@ {<<"opentelemetry_api">>, <<"91353EE40583B1D4F07D7B13ED62642ABFEC6AAA0D8A2114F07EDAFB2DF781C5">>}, {<<"opentelemetry_cowboy">>, <<"CE16BE94932DFCF9A036DE0B516413A5180AA6A4F31AD4073DD86EABE73F8837">>}, {<<"opentelemetry_exporter">>, <<"19A102D1F04776399A915BE27121852468318C27146E553FAF28008E3E474972">>}, + {<<"opentelemetry_logger_metadata">>, <<"0D1B7A4669521D75B142C9C9585771ED0707CCD515312CF06135C5BFC5C60AB9">>}, + {<<"opentelemetry_process_propagator">>, <<"81EC6825971903486EE73BE23230D06764DF39EE11011E520F601AA2BB21C893">>}, {<<"opentelemetry_telemetry">>, <<"BA1DF62515AED63F99A80DDF17E7A3873D1F686F23598EDEBF1633942772856E">>}, {<<"p1_utils">>, <<"7F94466ADA69BD982EA7BB80FBCA18E7053E7D0B82C9D9E37621FA508587069B">>}, {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, + {<<"prometheus">>, <<"B88F24279DD7A1F512CB090595FF6C88B50AAD0A6B394A4C4983725723DCD834">>}, + {<<"prometheus_cowboy">>, <<"CFCE0BC7B668C5096639084FCD873826E6220EA714BF60A716F5BD080EF2A99C">>}, + {<<"prometheus_httpd">>, <<"F616ED9B85B536B195D94104063025A91F904A4CFC20255363F49A197D96C896">>}, + {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>}, {<<"quickrand">>, <<"168CA3A8466A26912B8C3A1D6AA58975E1BB49E5C7AFB4998B80F6B90F910490">>}, {<<"ra">>, <<"C1AD68DE00B5DD3B32E6A30E1359AC676C3CAE47EECBA951789B470CBD4E1087">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, {<<"recon">>, <<"CBA53FA8DB83AD968C9A652E09C3ED7DDCC4DA434F27C3EAA9CA47FFB2B1FF03">>}, {<<"rfc3339">>, <<"1552DF616ACA368D982E9F085A0E933B6688A3F4938A671798978EC2C0C58730">>}, {<<"rocksdb">>, <<"5D23319998A7FCE5FFD5D7824116C905CABA7F91BAF8EDDABD0180F1BB272CEF">>}, + {<<"sbroker">>, <<"28FF1B5E58887C5098539F236307B36FE1D3EDAA2ACFF9D6A3D17C2DCAFEBBD0">>}, {<<"sext">>, <<"90A95B889F5C781B70BBCF44278B763148E313C376B60D87CE664CB1C1DD29B5">>}, {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}, {<<"telemetry">>, <<"A589817034A27EAB11144AD24D5C0F9FAB1F58173274B1E9BAE7074AF9CBEE51">>}, @@ -198,6 +232,7 @@ {<<"uuid">>, <<"77C3E3EE1E1701A2856CE945846D7CEB71931C60633A305D0B0FEAE03B2B3B5C">>}, {<<"yamerl">>, <<"4FF81FEE2F1F6A46F1700C0D880B24D193DDB74BD14EF42CB0BCF46E81EF2F8E">>}]}, {pkg_hash_ext,[ + {<<"accept">>, <<"11B18C220BCC2EAB63B5470C038EF10EB6783BCB1FCDB11AA4137DEFA5AC1BB8">>}, {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>}, {<<"aten">>, <<"8B623C8BE27B59A911D16AB0AF41777B504C147BC0D60A29015FAB58321C04B0">>}, {<<"base32">>, <<"10A73951D857D8CB1ECEEA8EB96C6941F6A76E105947AD09C2B73977DEE07638">>}, @@ -221,6 +256,7 @@ {<<"hackney">>, <<"DE16FF4996556C8548D512F4DBE22DD58A587BF3332E7FD362430A7EF3986B16">>}, {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, + {<<"inet_cidr">>, <<"3505F5DFAC7D862806C7051A3DD475363A45BCCF39CA1FAEE8EDA6A6B33CF335">>}, {<<"iso8601">>, <<"A8B00594F4309A41D17BA4AEAB2B94DFE1F4BE99F263BC1F46DAC9002CE99A29">>}, {<<"jsone">>, <<"A6C1DF6081DF742068D2ED747A4FE8A7740C56421B53E02BC9D4907DD3502922">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, @@ -234,15 +270,22 @@ {<<"opentelemetry_api">>, <<"2A8247F85C44216B883900067478D59955D11E58E5CFCA7C884CD4F203ACE3AC">>}, {<<"opentelemetry_cowboy">>, <<"455263DD177ACB8724391A5F2AF809E99F5D4A6DC71684E34A462F4D4AD02BD0">>}, {<<"opentelemetry_exporter">>, <<"43DE904DD7F482009BF1A40D591AB5ED25F603F1072A04A162E87F7F8C08DBB5">>}, + {<<"opentelemetry_logger_metadata">>, <<"772976D3C59651CF9C4600EDC238BB9CADF7F5EDAED1A1C5C59BF3E773DFE9FC">>}, + {<<"opentelemetry_process_propagator">>, <<"0572F26066BBB0457E22E169F966C0140A8F95237716C9C6BA4458D6DBAA724B">>}, {<<"opentelemetry_telemetry">>, <<"480F4FA1E992D597F931E7BC9E68478E8D904AD84489D2C5CA6EB6D48BBD7801">>}, {<<"p1_utils">>, <<"47F21618694EEEE5006AF1C88731AD86B757161E7823C29B6F73921B571C8502">>}, {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, + {<<"prometheus">>, <<"C3ABD6521E52CEC4F0D8ECA603CF214DFC84D8A27AA85946639F1424B8554D98">>}, + {<<"prometheus_cowboy">>, <<"BA286BECA9302618418892D37BCD5DC669A6CC001F4EB6D6AF85FF81F3F4F34C">>}, + {<<"prometheus_httpd">>, <<"0BBE831452CFDF9588538EB2F570B26F30C348ADAE5E95A7D87F35A5910BCF92">>}, + {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>}, {<<"quickrand">>, <<"4CB18E9304CF28E054E8DC6E151D1AC7F174E6FE31D5C1A07F71279B92A90800">>}, {<<"ra">>, <<"75C664334D4F294327DD8AAD06385294CC5CCE4763FBEF13EBF5075E930CEDF6">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, {<<"recon">>, <<"2C7523C8DEE91DFF41F6B3D63CBA2BD49EB6D2FE5BF1EEC0DF7F87EB5E230E1C">>}, {<<"rfc3339">>, <<"986D7F9BAC6891AA4D5051690058DE4E623245620BBEADA7F239F85C4DF8F23C">>}, {<<"rocksdb">>, <<"A4BDC5DD80ED137161549713062131E8240523787EBE7B51DF61CFB48B1786CE">>}, + {<<"sbroker">>, <<"BA952BFA35B374E1E5D84BC5F5EFE8360C6F99DC93B3118F714A9A2DFF6C9E19">>}, {<<"sext">>, <<"BC6016CB8690BAF677EACACFE6E7CADFEC8DC7E286CBBED762F6CD55B0678E73">>}, {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}, {<<"telemetry">>, <<"B727B2A1F75614774CFF2D7565B64D0DFA5BD52BA517F16543E6FC7EFCC0DF48">>}, -- cgit v1.2.3