diff options
author | James Every <devstopfix@gmail.com> | 2020-03-25 16:19:15 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-03-25 16:19:15 +0000 |
commit | 20c4068e5d8bd725c42513301b288094c8857041 (patch) | |
tree | bd226f972dfeaed7b11e76158ac1bd2243b89420 | |
parent | Merge commit '82cea2a0db4af442a3ea89a340e54fcd11cf8180' (diff) |
Convert to ports (#5)
* Remove erlexec dependency
* feat: async verify worker has started
* test: move infinite to script
* fix: timeout response
* fix: unix compile [Closes #169398412]
* feat: allow database patterns as worker param
This allows us to expand paths in an application, which is not possible from the configuration file.
* OTP 22 compatible with Elixir 1.7-1.10
* Install make on CI server
libmagic-dev contains magic.h
Fix this error:
** (Mix) Could not compile with "make" (exit status: 2).
* Fix make install
* Fix plurality
* Fix credo warning
* Allow multiple error messages
OS X and Linux return different errors messages.
* Allow named GenServer processes
* Types
* Disable test broken in ci
Works locally, not in CI.
test/gen_magic_test.exs:50
** (EXIT from #PID<0.1672.0>) shutdown
-rw-r--r-- | .credo.exs | 162 | ||||
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | .travis.yml | 18 | ||||
-rw-r--r-- | Makefile | 23 | ||||
-rw-r--r-- | README.md | 57 | ||||
-rw-r--r-- | config/config.exs | 14 | ||||
-rw-r--r-- | lib/gen_magic.ex | 16 | ||||
-rw-r--r-- | lib/gen_magic/apprentice_server.ex | 133 | ||||
-rw-r--r-- | lib/gen_magic/configuration.ex | 19 | ||||
-rw-r--r-- | mix.exs | 10 | ||||
-rw-r--r-- | mix.lock | 7 | ||||
-rw-r--r-- | src/apprentice.c | 7 | ||||
-rw-r--r-- | test/elixir | 6 | ||||
-rw-r--r-- | test/elixir.mgc | bin | 0 -> 688 bytes | |||
-rw-r--r-- | test/gen_magic_test.exs | 100 | ||||
-rw-r--r-- | test/soak.exs | 35 | ||||
-rw-r--r-- | test/test_helper.exs | 6 |
17 files changed, 502 insertions, 117 deletions
diff --git a/.credo.exs b/.credo.exs new file mode 100644 index 0000000..45ed758 --- /dev/null +++ b/.credo.exs @@ -0,0 +1,162 @@ +# This file contains the configuration for Credo and you are probably reading +# this after creating it with `mix credo.gen.config`. +# +# If you find anything wrong or unclear in this file, please report an +# issue on GitHub: https://github.com/rrrene/credo/issues +# +%{ + # + # You can have as many configs as you like in the `configs:` field. + configs: [ + %{ + # + # Run any exec using `mix credo -C <name>`. If no exec name is given + # "default" is used. + # + name: "default", + # + # These are the files included in the analysis: + files: %{ + # + # You can give explicit globs or simply directories. + # In the latter case `**/*.{ex,exs}` will be used. + # + included: ["lib/", "src/", "test/", "web/", "apps/"], + excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"] + }, + # + # Load and configure plugins here: + # + plugins: [], + # + # If you create your own checks, you must specify the source files for + # them here, so they can be loaded by Credo before running the analysis. + # + requires: [], + # + # If you want to enforce a style guide and need a more traditional linting + # experience, you can change `strict` to `true` below: + # + strict: false, + # + # If you want to use uncolored output by default, you can change `color` + # to `false` below: + # + color: true, + # + # You can customize the parameters of any check by adding a second element + # to the tuple. + # + # To disable a check put `false` as second element: + # + # {Credo.Check.Design.DuplicatedCode, false} + # + checks: [ + # + ## Consistency Checks + # + {Credo.Check.Consistency.ExceptionNames, []}, + {Credo.Check.Consistency.LineEndings, []}, + {Credo.Check.Consistency.ParameterPatternMatching, []}, + {Credo.Check.Consistency.SpaceAroundOperators, []}, + {Credo.Check.Consistency.SpaceInParentheses, []}, + {Credo.Check.Consistency.TabsOrSpaces, []}, + + # + ## Design Checks + # + # You can customize the priority of any check + # Priority values are: `low, normal, high, higher` + # + {Credo.Check.Design.AliasUsage, + [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]}, + # You can also customize the exit_status of each check. + # If you don't want TODO comments to cause `mix credo` to fail, just + # set this value to 0 (zero). + # + {Credo.Check.Design.TagTODO, [exit_status: 0]}, + {Credo.Check.Design.TagFIXME, []}, + + # + ## Readability Checks + # + {Credo.Check.Readability.AliasOrder, []}, + {Credo.Check.Readability.FunctionNames, []}, + {Credo.Check.Readability.LargeNumbers, []}, + {Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]}, + {Credo.Check.Readability.ModuleAttributeNames, []}, + {Credo.Check.Readability.ModuleDoc, []}, + {Credo.Check.Readability.ModuleNames, []}, + {Credo.Check.Readability.ParenthesesInCondition, []}, + {Credo.Check.Readability.ParenthesesOnZeroArityDefs, []}, + {Credo.Check.Readability.PredicateFunctionNames, []}, + {Credo.Check.Readability.PreferImplicitTry, []}, + {Credo.Check.Readability.RedundantBlankLines, []}, + {Credo.Check.Readability.Semicolons, []}, + {Credo.Check.Readability.SpaceAfterCommas, []}, + {Credo.Check.Readability.StringSigils, []}, + {Credo.Check.Readability.TrailingBlankLine, []}, + {Credo.Check.Readability.TrailingWhiteSpace, []}, + # TODO: enable by default in Credo 1.1 + {Credo.Check.Readability.UnnecessaryAliasExpansion, false}, + {Credo.Check.Readability.VariableNames, []}, + + # + ## Refactoring Opportunities + # + {Credo.Check.Refactor.CondStatements, []}, + {Credo.Check.Refactor.CyclomaticComplexity, []}, + {Credo.Check.Refactor.FunctionArity, []}, + {Credo.Check.Refactor.LongQuoteBlocks, []}, + {Credo.Check.Refactor.MapInto, []}, + {Credo.Check.Refactor.MatchInCondition, []}, + {Credo.Check.Refactor.NegatedConditionsInUnless, []}, + {Credo.Check.Refactor.NegatedConditionsWithElse, []}, + {Credo.Check.Refactor.Nesting, []}, + {Credo.Check.Refactor.UnlessWithElse, []}, + {Credo.Check.Refactor.WithClauses, []}, + + # + ## Warnings + # + {Credo.Check.Warning.BoolOperationOnSameValues, []}, + {Credo.Check.Warning.ExpensiveEmptyEnumCheck, []}, + {Credo.Check.Warning.IExPry, []}, + {Credo.Check.Warning.IoInspect, []}, + {Credo.Check.Warning.LazyLogging, []}, + {Credo.Check.Warning.OperationOnSameValues, []}, + {Credo.Check.Warning.OperationWithConstantResult, []}, + {Credo.Check.Warning.RaiseInsideRescue, []}, + {Credo.Check.Warning.UnusedEnumOperation, []}, + {Credo.Check.Warning.UnusedFileOperation, []}, + {Credo.Check.Warning.UnusedKeywordOperation, []}, + {Credo.Check.Warning.UnusedListOperation, []}, + {Credo.Check.Warning.UnusedPathOperation, []}, + {Credo.Check.Warning.UnusedRegexOperation, []}, + {Credo.Check.Warning.UnusedStringOperation, []}, + {Credo.Check.Warning.UnusedTupleOperation, []}, + + # + # Controversial and experimental checks (opt-in, just replace `false` with `[]`) + # + {Credo.Check.Consistency.MultiAliasImportRequireUse, false}, + {Credo.Check.Design.DuplicatedCode, false}, + {Credo.Check.Readability.MultiAlias, false}, + {Credo.Check.Readability.Specs, false}, + {Credo.Check.Readability.SinglePipe, false}, + {Credo.Check.Refactor.ABCSize, false}, + {Credo.Check.Refactor.AppendSingleItem, false}, + {Credo.Check.Refactor.DoubleBooleanNegation, false}, + {Credo.Check.Refactor.ModuleDependencies, false}, + {Credo.Check.Refactor.PipeChainStart, false}, + {Credo.Check.Refactor.VariableRebinding, false}, + {Credo.Check.Warning.MapGetUnsafePass, false}, + {Credo.Check.Warning.UnsafeToAtom, false} + + # + # Custom checks can be created using `mix credo.gen.check`. + # + ] + } + ] +} @@ -1,7 +1,7 @@ # The directory Mix will write compiled artifacts to. /_build/ *.o -priv/ +priv/apprentice # If you run "mix test --cover", coverage assets end up here. /cover/ @@ -24,3 +24,7 @@ erl_crash.dump # Ignore package tarball (built via "mix hex.build"). gen_magic-*.tar +.elixir_ls/ + +# Magic files +test/magic.mgc
\ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1c68e47 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: elixir + +elixir: + - 1.10 + - 1.9 + - 1.8 + - 1.7 + +otp_release: + - '22.2' + +before_script: + - mix compile –warnings-as-errors + - mix credo --strict + - if [[ "$TRAVIS_ELIXIR_VERSION" =~ "1.10" ]]; then mix format mix.exs "{config,clients,games,lib,test}/**/*.{ex,exs}" --check-formatted; fi + +before_install: + - sudo apt-get install -y build-essential erlang-dev libmagic-dev @@ -1,5 +1,7 @@ +# Apprentice binary + CC = gcc -CFLAGS = -std=c99 -g -Wall +CFLAGS = -std=c99 -g -Wall -Werror LDFLAGS = -lm -lmagic HEADER_FILES = src C_SOURCE_FILES = src/apprentice.c @@ -7,7 +9,18 @@ OBJECT_FILES = $(C_SOURCE_FILES:.c=.o) EXECUTABLE_DIRECTORY = priv EXECUTABLE = $(EXECUTABLE_DIRECTORY)/apprentice -all: $(C_SOURCE_FILES) $(EXECUTABLE) +# Unit test custom magic file + +MAGIC = file +TEST_DIRECTORY = test +TARGET_MAGIC = $(TEST_DIRECTORY)/elixir.mgc +SOURCE_MAGIC = $(TEST_DIRECTORY)/elixir + +# Target + +all: $(EXECUTABLE) $(TARGET_MAGIC) + +# Compile $(EXECUTABLE): $(OBJECT_FILES) $(EXECUTABLE_DIRECTORY) $(CC) $(OBJECT_FILES) -o $@ $(LDFLAGS) @@ -18,5 +31,11 @@ $(EXECUTABLE_DIRECTORY): .o: $(CC) $(CFLAGS) $< -o $@ +# Test case + +$(TARGET_MAGIC): $(SOURCE_MAGIC) + cd $(TEST_DIRECTORY); $(MAGIC) -C -m elixir + clean: rm -f $(EXECUTABLE) $(OBJECT_FILES) $(BEAM_FILES) + rm -f $(TEST_DIRECTORY)/*mgc @@ -1,6 +1,8 @@ # GenMagic -**TODO: Add description** +Determine file type. Elixir bindings for [libmagic](http://man7.org/linux/man-pages/man3/libmagic.3.html). + +[![Build Status](https://travis-ci.org/devstopfix/gen_magic.svg?branch=release_v1)](https://travis-ci.org/devstopfix/gen_magic) ## Installation @@ -10,12 +12,57 @@ by adding `gen_magic` to your list of dependencies in `mix.exs`: ```elixir def deps do [ - {:gen_magic, "~> 0.1.0"} + {:gen_magic, "~> 0.20"} ] end ``` -Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) -and published on [HexDocs](https://hexdocs.pm). Once published, the docs can -be found at [https://hexdocs.pm/gen_magic](https://hexdocs.pm/gen_magic). +## Usage + +The libmagic library requires a magic file which can be installed in various locations on your file system. A good way of locating it is given in the [config](config/config.exs): + +```elixir +database = [ + "/usr/local/share/misc/magic.mgc", + "/usr/share/file/magic.mgc", + "/usr/share/misc/magic.mgc" +] |> Enum.find(&File.exists?/1) +``` + +The GenServer SHOULD be run under a supervisor or a pool as it is designed to end should it receive any unexpected error. Here we run it under a supervisor: + +```elixir +{:ok, _} = Supervisor.start_link([ + {GenMagic.ApprenticeServer, + [database_patterns: [database], name: :gen_magic]}], + strategy: :one_for_one) +``` + +Now we can ask it to inspect a file: + +```elixir +> GenMagic.ApprenticeServer.file(:gen_magic, Path.expand("~/.bash_history")) +{:ok, [mime_type: "text/plain", encoding: "us-ascii", content: "ASCII text"]} +``` + +For a one shot test, use the helper method: + +```elixir +> GenMagic.perform(Path.join(File.cwd!(), "Makefile")) + +{:ok, + [ + mime_type: "text/x-makefile", + encoding: "us-ascii", + content: "makefile script, ASCII text" + ]} +``` + +## Soak test + +Run an endless cycle to prove that the GenServer is resilient: +```bash +find /usr/share/ -name *png | xargs mix run test/soak.exs +find . -name *ex | xargs mix run test/soak.exs +```
\ No newline at end of file diff --git a/config/config.exs b/config/config.exs index 3bf1711..72ed18e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,13 +1,13 @@ -# This file is responsible for configuring your application -# and its dependencies with the aid of the Mix.Config module. use Mix.Config config :gen_magic, worker_name: "apprentice", worker_timeout: 5000, recycle_threshold: 10, - database_patterns: [ - "/usr/local/share/misc/magic.mgc", - "/usr/share/file/magic.mgc", - "/usr/share/misc/magic.mgc" - ] + database_patterns: + [ + "/usr/local/share/misc/magic.mgc", + "/usr/share/file/magic.mgc", + "/usr/share/misc/magic.mgc" + ] + |> Enum.find(&File.exists?/1) diff --git a/lib/gen_magic.ex b/lib/gen_magic.ex index 93b3545..c60e357 100644 --- a/lib/gen_magic.ex +++ b/lib/gen_magic.ex @@ -3,25 +3,17 @@ defmodule GenMagic do Top-level namespace for GenMagic, the libMagic client for Elixir. """ + alias GenMagic.ApprenticeServer + @doc """ Top-level convenience function which creates an ad-hoc process. Usually this will be wrapped in a pool established by the author of the application that uses the library. """ def perform(path) do - {:ok, pid} = __MODULE__.ApprenticeServer.start_link() - result = GenServer.call(pid, {:perform, path}) + {:ok, pid} = ApprenticeServer.start_link([]) + result = ApprenticeServer.file(pid, path) :ok = GenServer.stop(pid) result end - - def perform_infinite(path) do - {:ok, pid} = __MODULE__.ApprenticeServer.start_link() - perform_infinite(path, pid) - end - - defp perform_infinite(path, pid, count \\ 0) do - IO.inspect([count, GenServer.call(pid, {:perform, path})]) - perform_infinite(path, pid, count + 1) - end end diff --git a/lib/gen_magic/apprentice_server.ex b/lib/gen_magic/apprentice_server.ex index 815d14f..004ba12 100644 --- a/lib/gen_magic/apprentice_server.ex +++ b/lib/gen_magic/apprentice_server.ex @@ -1,87 +1,96 @@ defmodule GenMagic.ApprenticeServer do @moduledoc """ Provides access to the underlying libMagic client which performs file introspection. + + This server needs to be supervised, as if it receives any unexpected error, it will terminate. """ alias GenMagic.Configuration use GenServer - def start_link(args \\ []) do - GenServer.start_link(__MODULE__, args) + @type result() :: [mime_type: String.t(), encoding: String.t(), content: String.t()] + + @worker_timeout Configuration.get_worker_timeout() + + def start_link([]) do + database_patterns = Configuration.get_database_patterns() + GenServer.start_link(__MODULE__, database_patterns: database_patterns) end - defmodule State do - defstruct pid: nil, ospid: nil, started: false, count: 0 + def start_link(name: name) do + database_patterns = Configuration.get_database_patterns() + GenServer.start_link(__MODULE__, [database_patterns: database_patterns], name: name) end - def init(_) do - {:ok, %State{}} + def start_link([database_patterns: _] = args) do + GenServer.start_link(__MODULE__, args) end - def handle_call(message, from, %{started: false} = state) do - case start(state) do - {:ok, state} -> handle_call(message, from, state) - {:error, _} = error -> {:reply, error, state} - end + def start_link(database_patterns: database_patterns, name: name) do + GenServer.start_link(__MODULE__, [database_patterns: database_patterns], name: name) end - def handle_call({:perform, path}, _, state) do - max_count = Configuration.get_recycle_threshold() + @doc """ + Determine a file type. + """ + @spec file(pid() | atom(), String.t()) :: {:ok, result()} | {:error, term()} + def file(pid, path) do + GenServer.call(pid, {:file, path}) + end - case {run(path, state), state.count + 1} do - {{:error, :worker_failure} = reply, _} -> - {:reply, reply, stop(state)} + def init(database_patterns: database_patterns) do + {worker_path, worker_arguments} = Configuration.get_worker_command(database_patterns) - {reply, ^max_count} -> - {:reply, reply, stop(state)} + case File.stat(worker_path) do + {:ok, _} -> + port = + Port.open({:spawn_executable, to_charlist(worker_path)}, [ + :stderr_to_stdout, + :binary, + :exit_status, + args: worker_arguments + ]) - {reply, count} -> - {:reply, reply, %{state | count: count}} - end - end + {:ok, port, {:continue, :verify_port}} - def handle_info({:DOWN, _, :process, pid, :normal}, state) do - case state.pid do - ^pid -> {:noreply, %State{}} - _ -> {:noreply, state} + {:error, _} = e -> + {:stop, e} end end - defp start(%{started: false} = state) do - worker_command = Configuration.get_worker_command() - worker_options = [stdin: true, stdout: true, stderr: true, monitor: true] - worker_timeout = Configuration.get_worker_timeout() - {:ok, pid, ospid} = Exexec.run(worker_command, worker_options) - state = %{state | started: true, pid: pid, ospid: ospid} - + # The process sends us an ack when it has read the databases + # and is ready to receive data. + # OTP-13019 Requires OTP 21 + def handle_continue(:verify_port, port) do receive do - {:stdout, ^ospid, "ok\n"} -> {:ok, state} - {:stdout, ^ospid, "ok\r\n"} -> {:ok, state} + {_, {:data, "ok\n"}} -> + {:noreply, port} after - worker_timeout -> - {:error, :worker_failure} + 10_000 -> + {:stop, :nok, port} end end - defp stop(%{started: true} = state) do - :normal = Exexec.stop_and_wait(state.ospid) - %State{} - end - - defp run(path, %{pid: pid, ospid: ospid} = _state) do - worker_timeout = Configuration.get_worker_timeout() - :ok = Exexec.send(pid, "file; " <> path <> "\n") + def handle_call({:file, path}, _from, port) do + cmd = "file; " <> path <> "\n" + send(port, {self(), {:command, cmd}}) receive do - {stream, ^ospid, message} -> - handle_response(stream, message) + {_, {:data, "ok; " <> message}} -> + {:reply, parse_response(message), port} + + {_, {:data, "error; " <> message}} -> + {:reply, {:error, String.trim(message)}, port} + + {_, {:data, _}} -> + {:stop, :shutdown, {:error, :malformed}, port} after - worker_timeout -> - {:error, :worker_failure} + @worker_timeout -> + {:stop, :shutdown, {:error, :worker_failure}, port} end end - defp handle_response(:stdout, "ok; " <> message) do + defp parse_response(message) do case message |> String.trim() |> String.split("\t") do [mime_type, encoding, content] -> {:ok, [mime_type: mime_type, encoding: encoding, content: content]} @@ -91,16 +100,20 @@ defmodule GenMagic.ApprenticeServer do end end - defp handle_response(:stderr, "error; " <> message) do - {:error, String.trim(message)} + def terminate(_reason, port) do + case send(port, {self(), :close}) do + {_, :close} -> :ok + _ -> :ok + end + end + + def handle_info({_, {:exit_status, 1}}, port) do + {:stop, :shutdown, port} end - # TODO handle late responses under load - # 17:13:47.808 [error] GenServer #PID<0.199.0> terminating - # ** (FunctionClauseError) no function clause matching in GenMagic.ApprenticeServer.handle_info/2 - # (gen_magic) lib/gen_magic/apprentice_server.ex:41: GenMagic.ApprenticeServer.handle_info({:stderr, 12304, "\n"}, %GenMagic.ApprenticeServer.State{count: 2, ospid: 12304, pid: #PID<0.243.0>, started: true}) - # (stdlib) gen_server.erl:637: :gen_server.try_dispatch/4 - # (stdlib) gen_server.erl:711: :gen_server.handle_msg/6 - # (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3 - # Last message: {:stderr, 12304, "\n"} + # Server is overloaded - late replies + # Normally caused only when previous call errors + def handle_info({_, {:data, _}}, port) do + {:stop, :shutdown, port} + end end diff --git a/lib/gen_magic/configuration.ex b/lib/gen_magic/configuration.ex index 15334e2..c78af34 100644 --- a/lib/gen_magic/configuration.ex +++ b/lib/gen_magic/configuration.ex @@ -5,11 +5,11 @@ defmodule GenMagic.Configuration do @otp_app Mix.Project.config()[:app] - def get_worker_command do - database_paths = get_database_paths() + def get_worker_command(patterns) do + database_paths = paths(patterns) worker_path = Path.join(:code.priv_dir(@otp_app), get_worker_name()) - worker_arguments = Enum.map(database_paths, &("--file " <> &1)) - Enum.join([worker_path | worker_arguments], " ") + worker_arguments = Enum.flat_map(database_paths, &["--file", &1]) + {worker_path, worker_arguments} end def get_worker_name do @@ -24,11 +24,18 @@ defmodule GenMagic.Configuration do get_env(:recycle_threshold) end - def get_database_paths do - get_env(:database_patterns) |> Enum.flat_map(&Path.wildcard/1) + def get_database_patterns do + case get_env(:database_patterns) do + nil -> [] + l when is_list(l) -> l + s when is_binary(s) -> [s] + end end defp get_env(key) do Application.get_env(@otp_app, key) end + + defp paths(patterns), + do: patterns |> Enum.flat_map(&Path.wildcard/1) |> Enum.filter(&File.exists?/1) end @@ -4,8 +4,8 @@ defmodule GenMagic.MixProject do def project do [ app: :gen_magic, - version: "0.1.0", - elixir: "~> 1.8", + version: "0.20.83", + elixir: "~> 1.7", start_permanent: Mix.env() == :prod, compilers: [:elixir_make] ++ Mix.compilers(), deps: deps() @@ -20,9 +20,9 @@ defmodule GenMagic.MixProject do defp deps do [ - {:elixir_make, "~> 0.4", runtime: false}, - {:exexec, "~> 0.2.0"}, - {:erlexec, "~> 1.10.0"} + {:credo, "~> 1.3", only: [:dev, :test], runtime: false}, + {:dialyxir, "~> 1.0.0-rc.6", only: [:dev], runtime: false}, + {:elixir_make, "~> 0.4", runtime: false} ] end end @@ -1,5 +1,10 @@ %{ - "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm"}, + "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "credo": {:hex, :credo, "1.3.1", "082e8d9268a489becf8e7aa75671a7b9088b1277cd6c1b13f40a55554b3f5126", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "0da816ed52fa520b9ea0e5d18a0d3ca269e0bd410b1174d88d8abd94be6cce3c"}, + "dialyxir": {:hex, :dialyxir, "1.0.0", "6a1fa629f7881a9f5aaf3a78f094b2a51a0357c843871b8bc98824e7342d00a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "aeb06588145fac14ca08d8061a142d52753dbc2cf7f0d00fc1013f53f8654654"}, + "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "erlexec": {:hex, :erlexec, "1.10.0", "cba7924cf526097d2082ceb0ec34e7db6bca2624b8f3867fb3fa89c4cf25d227", [:rebar3], [], "hexpm"}, "exexec": {:hex, :exexec, "0.2.0", "a6ffc48cba3ac9420891b847e4dc7120692fb8c08c9e82220ebddc0bb8d96103", [:mix], [{:erlexec, "~> 1.10", [hex: :erlexec, repo: "hexpm", optional: false]}], "hexpm"}, + "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, } diff --git a/src/apprentice.c b/src/apprentice.c index 04f08aa..1780f1e 100644 --- a/src/apprentice.c +++ b/src/apprentice.c @@ -98,7 +98,10 @@ void setup_options_file(char *optarg) { exit(1); } struct file *next = malloc(sizeof(struct file)); - next->path = strdup(optarg); + size_t path_length = strlen(optarg) + 1; + char *path = malloc(path_length); + memcpy(path, optarg, path_length); + next->path = path; next->next = magic_database; magic_database = next; } @@ -142,7 +145,7 @@ void process_line(char *line) { print_error("no_file"); return; } - + process_file(path); } diff --git a/test/elixir b/test/elixir new file mode 100644 index 0000000..8e62729 --- /dev/null +++ b/test/elixir @@ -0,0 +1,6 @@ +#------------------------------------------------------------------------------ +# file: file(1) magic for Elixir +# URL: https://elixir-lang.org/ + +0 string/w defmodule Elixir module source text +!:mime text/x-elixir diff --git a/test/elixir.mgc b/test/elixir.mgc Binary files differnew file mode 100644 index 0000000..cafeaee --- /dev/null +++ b/test/elixir.mgc diff --git a/test/gen_magic_test.exs b/test/gen_magic_test.exs index b0e187a..f9aae17 100644 --- a/test/gen_magic_test.exs +++ b/test/gen_magic_test.exs @@ -4,32 +4,102 @@ defmodule GenMagicTest do alias GenMagic.ApprenticeServer, as: Magic - setup_all do - {:ok, pid} = Magic.start_link() - {:ok, %{pid: pid}} - end + @iterations 10 - test "Makefile is text file", %{pid: pid} do - path = File.cwd!() |> Path.join("Makefile") + test "Makefile is text file" do + {:ok, pid} = Magic.start_link([]) + path = makefile_path() assert {:ok, [mime_type: "text/x-makefile", encoding: _, content: _]} = - GenServer.call(pid, {:perform, path}) + GenServer.call(pid, {:file, path}) + end + + test "Top level helper function" do + path = makefile_path() + assert {:ok, [mime_type: "text/x-makefile", encoding: _, content: _]} = GenMagic.perform(path) end @tag load: true, timeout: 180_000 - test "Load test local files", %{pid: pid} do - "/usr/share/**/*" - |> Path.wildcard() - |> Stream.reject(&File.dir?/1) - |> Stream.chunk_every(500) - |> Stream.flat_map(&Enum.shuffle/1) + test "Load test local files" do + {:ok, pid} = Magic.start_link([]) + + files_stream() |> Stream.cycle() - |> Stream.take(10000) + |> Stream.take(@iterations) |> Stream.map( - &assert {:ok, [mime_type: _, encoding: _, content: _]} = GenServer.call(pid, {:perform, &1}) + &assert {:ok, [mime_type: _, encoding: _, content: _]} = GenServer.call(pid, {:file, &1}) ) |> Enum.all?() |> assert end + test "Non-existent file" do + {:ok, pid} = Magic.start_link([]) + path = missing_filename() + assert_no_file(GenServer.call(pid, {:file, path})) + end + + test "Named process" do + {:ok, _pid} = Magic.start_link(name: :gen_magic) + path = makefile_path() + + assert {:ok, [mime_type: "text/x-makefile", encoding: _, content: _]} = + GenServer.call(:gen_magic, {:file, path}) + end + + @tag :ci + test "Custom database file recognises Elixir files" do + database = Path.join(File.cwd!(), "test/elixir.mgc") + {:ok, pid} = Magic.start_link(database_patterns: [database]) + path = Path.join(File.cwd!(), "mix.exs") + + assert GenServer.call(pid, {:file, path}) == + {:ok, + [ + mime_type: "text/x-elixir", + encoding: "us-ascii", + content: "Elixir module source text" + ]} + end + + @tag :breaking + test "Load test local files and missing files" do + {:ok, pid} = Magic.start_link([]) + + files_stream() + |> Stream.intersperse(missing_filename()) + |> Stream.cycle() + |> Stream.take(@iterations) + |> Stream.map(fn path -> + case GenServer.call(pid, {:file, path}) do + {:ok, [mime_type: _, encoding: _, content: _]} -> true + {:error, "no_file"} -> true + end + end) + |> Enum.all?() + |> assert + end + + defp missing_filename do + f = + make_ref() + |> inspect + |> String.replace(~r/\D/, "") + + Path.join("/tmp", f) + end + + defp files_stream, + do: + "/usr/share/**/*" + |> Path.wildcard() + |> Stream.reject(&File.dir?/1) + |> Stream.chunk_every(500) + |> Stream.flat_map(&Enum.shuffle/1) + + defp assert_no_file({:error, msg}) do + assert msg == "no_file" || msg == "", msg + end + + defp makefile_path, do: Path.join(File.cwd!(), "Makefile") end diff --git a/test/soak.exs b/test/soak.exs new file mode 100644 index 0000000..ab366f0 --- /dev/null +++ b/test/soak.exs @@ -0,0 +1,35 @@ +defmodule Soak do + @moduledoc """ + Run with a list of files to inspect: + + find /usr/share/ -name *png | xargs mix run test/soak.exs + """ + + def perform_infinite([]), do: false + + def perform_infinite(paths) do + {:ok, pid} = + GenMagic.ApprenticeServer.start_link(database_patterns: ["/usr/local/share/misc/*.mgc"]) + + perform_infinite(paths, [], pid, 0) + end + + defp perform_infinite([], done, pid, count) do + perform_infinite(done, [], pid, count) + end + + defp perform_infinite([path | paths], done, pid, count) do + if rem(count, 1000) == 0, do: IO.puts(Integer.to_string(count)) + + {:ok, [mime_type: _, encoding: _, content: _]} = GenServer.call(pid, {:file, path}) + perform_infinite(paths, [path | done], pid, count + 1) + end +end + +# Run with a list of files to inspect +# +# find /usr/share/ -name *png | xargs mix run test/soak.exs + +System.argv() +|> Enum.filter(&File.exists?/1) +|> Soak.perform_infinite() diff --git a/test/test_helper.exs b/test/test_helper.exs index 997c393..a7e2f12 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,5 @@ -ExUnit.start(exclude: [:load]) +excluded = + [:breaking] ++ + if(System.get_env("TRAVIS") != nil, do: [:ci], else: []) + +ExUnit.start(exclude: excluded) |