From 407b52120a6ed734c9c132844c8986723f1505ae Mon Sep 17 00:00:00 2001 From: Holger Just Date: Thu, 2 Feb 2017 15:04:02 +0100 Subject: [PATCH] Add basic fields to hold additional data on Buffers The fields follow the basic structure of basic Hashes and Arrays but provide an interface better suitable for us. Specifically: * They check and enforce the datatypes for keys and values to be strictly JSON-conforming. Only the basic data-types are accepted respectively converted to. * Hashes only accept String keys. * Basic values are always frozen. --- lib/rackstash.rb | 6 + lib/rackstash/fields.rb | 8 + lib/rackstash/fields/abstract_collection.rb | 143 +++++ lib/rackstash/fields/array.rb | 58 ++ lib/rackstash/fields/hash.rb | 100 ++++ .../fields/abstract_collection_spec.rb | 523 ++++++++++++++++++ spec/rackstash/fields/array_spec.rb | 148 +++++ spec/rackstash/fields/hash_spec.rb | 340 ++++++++++++ spec/rackstash_spec.rb | 11 + 9 files changed, 1337 insertions(+) create mode 100644 lib/rackstash/fields.rb create mode 100644 lib/rackstash/fields/abstract_collection.rb create mode 100644 lib/rackstash/fields/array.rb create mode 100644 lib/rackstash/fields/hash.rb create mode 100644 spec/rackstash/fields/abstract_collection_spec.rb create mode 100644 spec/rackstash/fields/array_spec.rb create mode 100644 spec/rackstash/fields/hash_spec.rb diff --git a/lib/rackstash.rb b/lib/rackstash.rb index 77274b9..3e3a315 100644 --- a/lib/rackstash.rb +++ b/lib/rackstash.rb @@ -3,6 +3,8 @@ # This software may be modified and distributed under the terms # of the MIT license. See the LICENSE.txt file for details. +require 'set' + require 'rackstash/version' module Rackstash @@ -18,6 +20,10 @@ module Rackstash PROGNAME = "rackstash/v#{Rackstash::VERSION}".freeze EMPTY_STRING = ''.freeze + EMPTY_SET = Set.new.freeze + + # How many decimal places to render on ISO 8601 timestamps + ISO8601_PRECISION = 3 end require 'rackstash/logger' diff --git a/lib/rackstash/fields.rb b/lib/rackstash/fields.rb new file mode 100644 index 0000000..1252d8f --- /dev/null +++ b/lib/rackstash/fields.rb @@ -0,0 +1,8 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'rackstash/fields/abstract_collection' +require 'rackstash/fields/hash' +require 'rackstash/fields/array' diff --git a/lib/rackstash/fields/abstract_collection.rb b/lib/rackstash/fields/abstract_collection.rb new file mode 100644 index 0000000..040169c --- /dev/null +++ b/lib/rackstash/fields/abstract_collection.rb @@ -0,0 +1,143 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'bigdecimal' +require 'pathname' +require 'uri' + +module Rackstash + module Fields + class AbstractCollection + def ==(other) + self.class == other.class && raw == other.raw + end + alias :eql? :== + + def inspect + "#<#{self.class}:#{format '0x%014x', object_id << 1} #{to_s}>" + end + + # Provide a copy of the wrapped {#raw} data in a format allowing direct + # mapping to JSON. + # + # @note This method is usually overridden in child classes. + def as_json(*) + nil + end + + def to_json(*) + as_json.to_json + end + + def to_s + as_json.inspect + end + + protected + + attr_accessor :raw + + private + + def initialize_copy(source) + super + self.raw = source.raw == nil ? nil : source.raw.dup + self + end + + def new(raw) + self.class.new.tap do |new_object| + new_object.raw = raw + end + end + + # @param str [#to_s] + def utf8_encode(str) + str.to_s.encode( + Encoding::UTF_8, + invalid: :replace, + undef: :replace + ) + end + + def resolve_value(value, scope: nil) + return value unless value.is_a?(Proc) + scope == nil ? value.call : scope.instance_exec(&value) + end + + # Note: You should never mutate an array or hash returned by normalize + # when `wrap` is `false`. + def normalize(value, resolve: true, scope: nil, wrap: true) + value = resolve_value(value, scope: scope) if resolve + + case value + when ::String, ::Symbol + return utf8_encode(value).freeze + when ::Integer, ::Float + return value + when true, false, nil + return value + when Rackstash::Fields::AbstractCollection + return wrap ? value : value.raw + when ::Hash + hash = value.each_with_object({}) do |(k, v), memo| + memo[utf8_encode(k)] = normalize(v, scope: scope, resolve: resolve) + end + hash = Rackstash::Fields::Hash.new.tap do |hash_field| + hash_field.raw = hash + end if wrap + return hash + when ::Array, ::Enumerator + array = value.map { |e| normalize(e, scope: scope, resolve: resolve) } + array = Rackstash::Fields::Array.new.tap do |array_field| + array_field.raw = array + end if wrap + return array + when ::Time + return value.utc.iso8601(ISO8601_PRECISION).freeze + when ::DateTime + return value.to_time.utc.iso8601(ISO8601_PRECISION).freeze + when ::Date + return value.iso8601.encode!(Encoding::UTF_8).freeze + when ::Regexp, ::Range, ::URI::Generic, ::Pathname + return utf8_encode(value).freeze + when Exception + exception = "#{value.message} (#{value.class})" + exception << "\n" << value.backtrace.join("\n") if value.backtrace + return utf8_encode(exception).freeze + when ::Proc + return resolve ? utf8_encode(value.inspect).freeze : value + when ::BigDecimal + # A BigDecimal would be naturally represented as a JSON number. Most + # libraries, however, parse non-integer JSON numbers directly as + # floats. Clients using those libraries would get in general a wrong + # number and no way to recover other than manually inspecting the + # string with the JSON code itself. + return value.to_s('F').encode!(Encoding::UTF_8).freeze + when ::Complex + # A complex number can not reliably converted to a float or rational, + # thus we always transform it to a String + return utf8_encode(value).freeze + end + + # Try to convert the value to a known basic type and recurse + %i[ + as_json + to_hash to_ary to_h to_a + to_time to_datetime to_date + to_f to_i + ].each do |method| + # Try to convert the value to a base type but ignore any errors + next unless value.respond_to?(method) + value = value.public_send(method) rescue next + + return normalize(value, scope: scope, wrap: wrap, resolve: resolve) + end + + utf8_encode(value.inspect).freeze + end + end + end +end diff --git a/lib/rackstash/fields/array.rb b/lib/rackstash/fields/array.rb new file mode 100644 index 0000000..cf1743c --- /dev/null +++ b/lib/rackstash/fields/array.rb @@ -0,0 +1,58 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'rackstash/fields/abstract_collection' + +module Rackstash + module Fields + class Array < AbstractCollection + def initialize + @raw = [] + end + + def [](index) + @raw[index] + end + + def []=(index, value) + @raw[index] = normalize(value) + end + + def as_json(*) + @raw.map { |value| + value.is_a?(AbstractCollection) ? value.as_json : value + } + end + alias :to_ary :as_json + alias :to_a :as_json + + def clear + @raw.clear + self + end + + def concat(array) + array = Array(normalize(array, wrap: false)) + @raw.concat(array) + self + end + + def length + @raw.length + end + + private + + def Array(obj) + return obj.to_ary if obj.respond_to?(:to_ary) + raise TypeError, "no implicit conversion of #{obj.class} into Array" + end + end + + def self.Array(array) + Array.new.concat(array) + end + end +end diff --git a/lib/rackstash/fields/hash.rb b/lib/rackstash/fields/hash.rb new file mode 100644 index 0000000..d5ccda7 --- /dev/null +++ b/lib/rackstash/fields/hash.rb @@ -0,0 +1,100 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'rackstash/fields/abstract_collection' + +module Rackstash + module Fields + class Hash < AbstractCollection + def initialize(forbidden_keys: EMPTY_SET) + @raw = {} + + if forbidden_keys.is_a?(Set) + forbidden_keys = forbidden_keys.dup.freeze unless forbidden_keys.frozen? + @forbidden_keys = forbidden_keys + else + @forbidden_keys = Set[*forbidden_keys].freeze + end + end + + def [](key) + @raw[utf8_encode(key)] + end + + def []=(key, value) + key = utf8_encode(key) + raise ArgumentError, "Forbidden field #{key}" if forbidden_key?(key) + + @raw[key] = normalize(value) + end + alias :store :[]= + + def as_json(*) + @raw.each_with_object({}) do |(key, value), memo| + value = value.as_json if value.is_a?(AbstractCollection) + memo[key] = value + end + end + alias :to_hash :as_json + alias :to_h :as_json + + def clear + @raw.clear + self + end + + def keys + @raw.keys + end + + def merge(hash, force: true, scope: nil, &block) + dup.merge!(hash, force: force, scope: scope, &block) + end + + def merge!(hash, force: true, scope: nil) + hash = Hash(normalize(hash, scope: scope, wrap: false)) + + if force + forbidden = @forbidden_keys & hash.keys + unless forbidden.empty? + raise ArgumentError, "Forbidden keys #{forbidden.to_a.join(', ')}" + end + else + hash = hash.reject { |k, _v| forbidden_key?(k) } + end + + if block_given? + @raw.merge!(hash) { |key, old_val, new_val| + yielded = yield(key, old_val, new_val) + normalize(yielded, scope: scope) + } + else + @raw.merge!(hash) + end + self + end + alias :update :merge! + + def forbidden_key?(key) + @forbidden_keys.include?(key) + end + + def values + @raw.values + end + + private + + def Hash(obj) + return obj.to_hash if obj.respond_to?(:to_hash) + raise TypeError, "no implicit conversion of #{obj.class} into Hash" + end + end + + def self.Hash(raw, forbidden_keys: EMPTY_SET) + Hash.new(forbidden_keys: forbidden_keys).merge!(raw) + end + end +end diff --git a/spec/rackstash/fields/abstract_collection_spec.rb b/spec/rackstash/fields/abstract_collection_spec.rb new file mode 100644 index 0000000..7fe1560 --- /dev/null +++ b/spec/rackstash/fields/abstract_collection_spec.rb @@ -0,0 +1,523 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'spec_helper' + +require 'rackstash/fields/abstract_collection' +require 'rackstash/fields/array' +require 'rackstash/fields/hash' + +describe Rackstash::Fields::AbstractCollection do + let(:collection) { Rackstash::Fields::AbstractCollection.new } + + def normalize(*args) + collection.send(:normalize, *args) + end + + describe '#to_json' do + it 'returns the JSON version of as_json' do + as_json = double('JSON value') + expect(collection).to receive(:as_json).and_return(as_json) + expect(as_json).to receive(:to_json) + + collection.to_json + end + end + + describe '#inspect' do + it 'formats the object' do + expect(collection).to receive(:as_json).and_return('beepboop') + expect(collection.inspect).to( + match %r{\A#\z} + ) + end + end + + describe '#eql?' do + it 'is equal with the same class with the same raw value' do + expect(collection).to receive(:eql?).twice.and_call_original + expect(collection).to receive(:==).twice.and_call_original + + other = Rackstash::Fields::AbstractCollection.new + expect(collection).to eql other + expect(collection).to eq other + + other.send(:raw=, 'different value') + expect(collection).not_to eql other + expect(collection).not_to eq other + end + + it 'is not equal on different classes' do + other = Struct.new(:raw).new + expect(collection.send(:raw)).to eql other.raw + + expect(collection).to receive(:eql?).and_call_original + expect(collection).not_to eql other + + expect(collection).to receive(:==).and_call_original + expect(collection).not_to eq other + end + end + + describe '#dup' do + it 'dups the raw value' do + raw = 'hello' + collection.send(:raw=, raw) + expect(collection.send(:raw)).to equal raw + + duped = collection.dup + + expect(duped).not_to equal collection + expect(duped.send(:raw)).to eql 'hello' + expect(duped.send(:raw)).not_to equal raw + end + end + + describe '#raw' do + it 'is a protected accessor' do + expect { collection.raw = nil }.to raise_error NoMethodError + expect { collection.raw }.to raise_error NoMethodError + + collection.send(:raw=, 'beep') + expect(collection.send(:raw)).to eql 'beep' + end + end + + describe '#normalize' do + describe 'with String' do + it 'transforms encoding to UTF-8' do + utf8_str = 'Dönerstraße' + latin_str = utf8_str.encode(Encoding::ISO8859_9) + expect(latin_str.encoding).to eql Encoding::ISO8859_9 + + expect(normalize(latin_str)).to eql utf8_str + expect(normalize(latin_str).encoding).to eql Encoding::UTF_8 + expect(normalize(latin_str)).to be_frozen + end + + it 'replaces invalid characters in correctly encoded strings' do + binary = Digest::SHA256.digest('string') + + expect(normalize(binary)).to include '�' + expect(normalize(binary).encoding).to eql Encoding::UTF_8 + expect(normalize(binary)).to be_frozen + end + + it 'replaces invalid characters in incorrectly encoded strings' do + strange = Digest::SHA256.digest('string').force_encoding(Encoding::UTF_8) + + expect(normalize(strange)).to include '�' + expect(normalize(strange).encoding).to eql Encoding::UTF_8 + expect(normalize(strange)).to be_frozen + end + + it 'does not alter valid strings' do + valid = 'Dönerstraße'.freeze + + expect(normalize(valid)).to eql valid + # All strings are still copied and frozen, even if they don't need to + # be re-encoded. + expect(normalize(valid)).not_to equal valid + expect(normalize(valid)).to be_frozen + end + end + + it 'transforms Symbol to String' do + symbol = :foo + + expect(normalize(symbol)).to eql 'foo' + expect(normalize(symbol).encoding).to eql Encoding::UTF_8 + expect(normalize(symbol)).to be_frozen + end + + it 'passes Integer' do + fixnum = 42 + expect(normalize(fixnum)).to equal fixnum + expect(normalize(fixnum)).to be_frozen + + bignum = 10**100 + expect(normalize(bignum)).to equal bignum + expect(normalize(bignum)).to be_frozen + end + + it 'passes Float' do + float = 123.456 + + expect(normalize(float)).to equal float + expect(normalize(float)).to be_frozen + end + + it 'passes true, false, nil' do + expect(normalize(true)).to equal true + expect(normalize(false)).to equal false + expect(normalize(nil)).to equal nil + end + + describe 'with Rackstash::Fields::AbstractCollection' do + let(:raw) { 'beepboop' } + let(:value) { + value = double('Rackstash::Fields::AbstractCollection') + allow(value).to receive(:raw).and_return raw + value + } + + before do + expect(Rackstash::Fields::AbstractCollection).to( + receive(:===).with(value).and_return(true) + ) + end + + it 'passes the collection by default' do + expect(normalize(value)).to equal value + end + + it 'unwraps the collection if selected' do + expect(normalize(value, wrap: false)).to equal raw + end + end + + describe 'with Hash' do + it 'wraps the hash in a Rackstash::Fields::Hash' do + hash = { 'beep' => 'boop' } + + expect(normalize(hash)).to be_a Rackstash::Fields::Hash + expect(normalize(hash).send(:raw)).to eql 'beep' => 'boop' + expect(normalize(hash).send(:raw)).to_not equal hash + end + + it 'normalizes keys to frozen UTF-8 strings' do + hash = { 1 => 1, :two => 2, 'three' => 3, nil => 4 } + + expect(normalize(hash, wrap: false)).to eql( + { '1' => 1, 'two' => 2, 'three' => 3, '' => 4 } + ) + expect(normalize(hash, wrap: false).keys).to all be_frozen + end + + it 'normalizes all values' do + hash = { 'key' => :beepboop } + + expect(collection).to receive(:normalize).with(hash).ordered + .twice.and_call_original + expect(collection).to receive(:normalize).with(:beepboop, anything).ordered + .twice.and_call_original + + expect(normalize(hash)).to be_a Rackstash::Fields::Hash + expect(normalize(hash).send(:raw)).to eql 'key' => 'beepboop' + end + + it 'deep-wraps the hash' do + hash = { beep: { rawr: 'growl' } } + + expect(normalize(hash)).to be_a Rackstash::Fields::Hash + expect(normalize(hash)['beep']).to be_a Rackstash::Fields::Hash + expect(normalize(hash)['beep']['rawr']).to eql 'growl' + end + + it 'copies the hash' do + raw_hash = { beep: 'boing' } + wrapped_hash = normalize(raw_hash) + + raw_hash['foo'] = 'bar' + expect(wrapped_hash['foo']).to be_nil + end + + describe 'with procs' do + it 'resolves values' do + hash = { beep: -> { { rawr: -> { 'growl' } } } } + + expect(normalize(hash)).to be_a Rackstash::Fields::Hash + expect(normalize(hash)['beep']).to be_a Rackstash::Fields::Hash + expect(normalize(hash)['beep']['rawr']).to eql 'growl' + end + + it 'resolves values with the supplied scope' do + scope = 'scope' + hash = { beep: -> { { self => -> { upcase } } } } + + expect(normalize(hash, scope: scope)).to be_a Rackstash::Fields::Hash + expect(normalize(hash, scope: scope)['beep']).to be_a Rackstash::Fields::Hash + expect(normalize(hash, scope: scope)['beep']['scope']).to eql 'SCOPE' + end + end + end + + describe 'with Array' do + it 'wraps the array in a Rackstash::Fields::Array' do + array = ['beep', 'boop'] + + expect(normalize(array)).to be_a Rackstash::Fields::Array + expect(normalize(array).send(:raw)).to eql ['beep', 'boop'] + expect(normalize(array).send(:raw)).to_not equal array + end + + it 'normalizes values to frozen UTF-8 strings' do + array = [1, :two, 'three', nil] + + expect(normalize(array, wrap: false)).to eql [1, 'two', 'three', nil] + expect(normalize(array, wrap: false)).to all be_frozen + end + + it 'normalizes all values' do + array = ['boop', :beep] + + expect(collection).to receive(:normalize).with(array).ordered + .twice.and_call_original + expect(collection).to receive(:normalize).with('boop', anything).ordered + .twice.and_call_original + expect(collection).to receive(:normalize).with(:beep, anything).ordered + .twice.and_call_original + + expect(normalize(array)).to be_a Rackstash::Fields::Array + expect(normalize(array).send(:raw)).to eql ['boop', 'beep'] + end + + it 'deep-wraps the array' do + array = [123, ['foo', :bar]] + + expect(normalize(array)).to be_a Rackstash::Fields::Array + expect(normalize(array)[0]).to eql 123 + expect(normalize(array)[1]).to be_a Rackstash::Fields::Array + expect(normalize(array)[1][0]).to eql 'foo' + expect(normalize(array)[1][1]).to eql 'bar' + end + + it 'copies the array' do + raw_array = [12, 'boing'] + wrapped_array = normalize(raw_array) + + raw_array[2] = 'foo' + expect(wrapped_array[2]).to be_nil + end + + describe 'with procs' do + it 'resolves values' do + array = [123, -> { ['foo', -> { :bar }] }] + + expect(normalize(array)).to be_a Rackstash::Fields::Array + expect(normalize(array)[1]).to be_a Rackstash::Fields::Array + expect(normalize(array)[1][1]).to eql 'bar' + end + + it 'resolves values with the supplied scope' do + scope = 'string'.freeze + array = [123, -> { [upcase, -> { self }] }] + + expect(normalize(array, scope: scope)[1][0]).to eql 'STRING' + expect(normalize(array, scope: scope)[1][1]).to eql scope + end + end + end + + it 'wraps an Enumerator in a Rackstash::Fields::Array' do + small_numbers = Enumerator.new do |y| + 3.times do |i| + y << i + end + end + expect(normalize(small_numbers)).to be_a Rackstash::Fields::Array + expect(normalize(small_numbers).send(:raw)).to eql [0, 1, 2] + end + + it 'formats Time as an ISO 8601 UTC timestamp' do + time = Time.parse('2016-10-17 15:37:42 CEST') # UTC +02:00 + + expect(normalize(time)).to eql '2016-10-17T13:37:42.000Z' + expect(normalize(time)).to be_frozen + end + + it 'formats Time as an ISO 8601 UTC timestamp' do + time = Time.parse('2016-10-17 15:37:42 CEST') # UTC +02:00 + + expect(normalize(time)).to eql '2016-10-17T13:37:42.000Z' + expect(normalize(time).encoding).to eql Encoding::UTF_8 + expect(normalize(time)).to be_frozen + end + + it 'formats DateTime as an ISO 8601 UTC timestamp' do + datetime = DateTime.parse('2016-10-17 15:37:42 CEST') # UTC +02:00 + + expect(normalize(datetime)).to eql '2016-10-17T13:37:42.000Z' + expect(normalize(datetime).encoding).to eql Encoding::UTF_8 + expect(normalize(datetime)).to be_frozen + end + + it 'formats Date as an ISO 8601 date string' do + date = Date.new(2016, 10, 17) + + expect(normalize(date)).to eql '2016-10-17' + expect(normalize(date).encoding).to eql Encoding::UTF_8 + expect(normalize(date)).to be_frozen + end + + it 'transforms Regexp to String' do + regexp = /.?|(..+?)\1+/ + + expect(normalize(regexp)).to eql '(?-mix:.?|(..+?)\1+)' + expect(normalize(regexp).encoding).to eql Encoding::UTF_8 + expect(normalize(regexp)).to be_frozen + end + + it 'transforms Range to String' do + range = (1..10) + + expect(normalize(range)).to eql '1..10' + expect(normalize(range).encoding).to eql Encoding::UTF_8 + expect(normalize(range)).to be_frozen + end + + it 'transforms URI to String' do + uris = { + URI('https://example.com/p/f.txt') => 'https://example.com/p/f.txt', + URI('') => '' + } + + uris.each do |uri, result| + expect(uri).to be_a URI::Generic + + expect(normalize(uri)).to eql result + expect(normalize(uri).encoding).to eql Encoding::UTF_8 + expect(normalize(uri)).to be_frozen + end + end + + it 'transforms URI to String' do + uris = { + URI('https://example.com/p/f.txt') => 'https://example.com/p/f.txt', + URI('') => '' + } + + uris.each do |uri, result| + expect(uri).to be_a URI::Generic + + expect(normalize(uri)).to eql result + expect(normalize(uri).encoding).to eql Encoding::UTF_8 + expect(normalize(uri)).to be_frozen + end + end + + it 'transforms Pathname to String' do + pathname = Pathname.new('/path/to/file.ext'.encode(Encoding::ISO8859_9)) + + expect(normalize(pathname)).to eql '/path/to/file.ext' + expect(normalize(pathname).encoding).to eql Encoding::UTF_8 + expect(normalize(pathname)).to be_frozen + end + + it 'formats an Exception with Backtrace' do + exception = nil + begin + raise StandardError, 'An Error' + rescue => e + exception = e + end + + checker = Regexp.new <<-EOF.gsub(/^\s+/, '').rstrip, Regexp::MULTILINE + \\AAn Error \\(StandardError\\) + #{Regexp.escape __FILE__}:#{__LINE__ - 7}:in `block .*` + EOF + expect(normalize(exception)).to match checker + expect(normalize(exception).encoding).to eql Encoding::UTF_8 + expect(normalize(exception)).to be_frozen + end + + it 'transforms BigDecimal to String' do + bigdecimal = BigDecimal.new('123.987653') + + expect(normalize(bigdecimal)).to eql '123.987653' + expect(normalize(bigdecimal).encoding).to eql Encoding::UTF_8 + expect(normalize(bigdecimal)).to be_frozen + end + + describe 'with Proc' do + it 'calls the proc by default and normalizes the result' do + proc = proc { :return } + + expect(normalize(proc)).to eql 'return' + expect(normalize(proc).encoding).to eql Encoding::UTF_8 + expect(normalize(proc)).to be_frozen + end + + it 'inspects a nested proc' do + inner = proc { :return } + outer = proc { inner } + + expect(normalize(outer)).to match %r{\A#\z} + end + + it 'returns the proc when not resolving' do + outer = proc { :return } + + expect(normalize(outer, resolve: false)).to equal outer + end + end + + it 'transforms Complex to String' do + complex = Complex(2, 3) + + expect(normalize(complex)).to eql '2+3i' + expect(normalize(complex).encoding).to eql Encoding::UTF_8 + expect(normalize(complex)).to be_frozen + end + + it 'transforms Rational to Float' do + rational = Rational(-8, 6) + + expect(normalize(rational)).to be_a Float + expect(normalize(rational)).to be_frozen + end + + context 'conversion methods' do + let(:methods) { + %i[as_json to_hash to_ary to_h to_a to_time to_datetime to_date to_f to_i] + } + + it 'attempts conversion to base objects in order' do + methods.each_with_index do |method, i| + obj = double("#{method} - successful") + + methods[0..i].each_with_index do |check, j| + expect(obj).to receive(:respond_to?).with(check).and_return(i==j) + .ordered.once + end + + expect(obj).to receive(method).and_return("obj with #{method}") + .ordered.once + expect(normalize(obj)).to eql "obj with #{method}" + end + end + + it 'falls back on conversion error' do + obj = double('erroneous') + + methods.each do |method| + expect(obj).to receive(:respond_to?).with(method).and_return(true) + .ordered.once + expect(obj).to receive(method).and_raise('foo').ordered.once + end + + expect(obj).to receive(:inspect).and_return 'finally' + expect(normalize(obj)).to eql 'finally' + end + + it 'inspects objects we don\'t have a special rule for' do + obj = double('any object') + expect(obj).to receive(:inspect).and_return('an object') + + expect(normalize(obj)).to eql 'an object' + end + end + end + + describe '#to_s' do + it 'inspects #as_json' do + as_json = double('JSON value') + expect(collection).to receive(:as_json).and_return(as_json) + expect(as_json).to receive(:inspect) + + collection.to_s + end + end +end diff --git a/spec/rackstash/fields/array_spec.rb b/spec/rackstash/fields/array_spec.rb new file mode 100644 index 0000000..bb2afd7 --- /dev/null +++ b/spec/rackstash/fields/array_spec.rb @@ -0,0 +1,148 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'spec_helper' + +require 'rackstash/fields/array' + +describe Rackstash::Fields::Array do + let(:array) { Rackstash::Fields::Array.new } + + describe '#[]' do + it 'returns nil if a value was not set' do + expect(array[1]).to be_nil + end + + it 'returns a set value' do + array[0] = 'value' + expect(array[0]).to eql 'value' + end + end + + describe '#[]=' do + it 'normalizes values' do + expect(array).to receive(:normalize).with('value').and_return('normalized') + + array[0] = 'value' + expect(array[0]).to eql 'normalized' + end + end + + describe '#as_json' do + before do + array[0] = 'value' + array[1] = { 'key' => 'nested value', number: 42 } + array[2] = ['v1', :v2] + end + + it 'returns a simple array' do + expect(array.as_json).to be_a ::Array + expect(array.as_json.length).to eql 3 + end + + it 'returns a nested hash' do + expect(array[1]).to be_a Rackstash::Fields::Hash + + expect(array.as_json[1]).to be_a Hash + expect(array.as_json[1]).to eql 'key' => 'nested value', 'number' => 42 + end + + it 'returns a nested array' do + expect(array[2]).to be_a Rackstash::Fields::Array + + expect(array.as_json[2]).to be_an ::Array + expect(array.as_json[2]).to eql %w[v1 v2] + end + + it 'returns a new copy each time' do + expect(array.as_json).to eql array.as_json + expect(array.as_json).not_to equal array.as_json + + expect(array.as_json[1]).to eql array.as_json[1] + expect(array.as_json[1]).not_to equal array.as_json[1] + + expect(array.as_json[2]).to eql array.as_json[2] + expect(array.as_json[2]).not_to equal array.as_json[2] + end + + it 'can not change the raw value' do + as_json = array.as_json + as_json[3] = 'foo' + + expect(array[3]).to be_nil + end + + it 'can use to_ary as an alias' do + expect(array.to_ary).to eql array.as_json + end + + it 'can use to_a as an alias' do + expect(array.to_a).to eql array.as_json + end + end + + describe '#clear' do + it 'clears the array' do + array[0] = 'beep' + array.clear + expect(array[0]).to be_nil + end + + it 'returns the array' do + array[0] = 'bar' + expect(array.clear).to equal array + end + end + + describe '#concat' do + it 'contacts an array' do + array[0] = 'first' + ary = ['foo', 'bar'] + + expect(array).to receive(:normalize).with(ary, anything).ordered.and_call_original + expect(array).to receive(:normalize).with('foo', anything).ordered.and_call_original + expect(array).to receive(:normalize).with('bar', anything).ordered.and_call_original + + expect(array.concat(ary)).to equal array + + expect(array[0]).to eql 'first' + + expect(array[1]).to eql 'foo' + expect(array[1]).to be_frozen + expect(array[2]).to eql 'bar' + expect(array[2]).to be_frozen + end + + it 'refuses to concat an arbitrary value' do + expect { array.concat(:foo) }.to raise_error TypeError + expect { array.concat(42) }.to raise_error TypeError + expect { array.concat(false) }.to raise_error TypeError + expect { array.concat(nil) }.to raise_error TypeError + end + end + + describe '#length' do + it 'returns the length of the array' do + expect(array.length).to eql 0 + + array[0] = 'first' + expect(array.length).to eql 1 + + array.clear + expect(array.length).to eql 0 + end + end + + describe 'Converter' do + it 'creates a new array' do + raw = [Time.now, 'foo'] + array = Rackstash::Fields::Array(raw) + + expect(array).to be_a Rackstash::Fields::Array + expect(array[0]).to be_a String + expect(array[1]).to eql 'foo' + end + end +end diff --git a/spec/rackstash/fields/hash_spec.rb b/spec/rackstash/fields/hash_spec.rb new file mode 100644 index 0000000..e755ea5 --- /dev/null +++ b/spec/rackstash/fields/hash_spec.rb @@ -0,0 +1,340 @@ +# Copyright 2017 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'spec_helper' + +require 'rackstash/fields/hash' + +describe Rackstash::Fields::Hash do + let(:forbidden_keys) { Set.new } + let(:hash) { Rackstash::Fields::Hash.new(forbidden_keys: forbidden_keys) } + + describe '#initialize' do + it 'can be initialized without any arguments' do + Rackstash::Fields::Hash.new + end + + it 'accepts forbidden_keys as an Array' do + hash = Rackstash::Fields::Hash.new(forbidden_keys: ['field']) + expect(hash.instance_variable_get('@forbidden_keys')).to be_a Set + end + + it 'accepts forbidden_keys as a Set' do + hash = Rackstash::Fields::Hash.new(forbidden_keys: Set['field']) + expect(hash.instance_variable_get('@forbidden_keys')).to be_a Set + end + end + + describe 'subscript accessors' do + it 'normalizes keys when setting values' do + hash[:foo] = 'foo value' + expect(hash['foo']).to eql 'foo value' + + hash[42] = '42 value' + expect(hash['42']).to eql '42 value' + + hash[nil] = 'nil value' + expect(hash['']).to eql 'nil value' + end + + it 'normalizes keys when accessing values' do + hash['foo'] = 'foo value' + expect(hash[:foo]).to eql 'foo value' + + hash['42'] = '42 value' + expect(hash[42]).to eql '42 value' + + hash[''] = 'nil value' + expect(hash[nil]).to eql 'nil value' + end + + it 'returns nil if a value was not set' do + expect(hash['missing']).to be_nil + end + + it 'normalizes values' do + value = 'value' + expect(hash).to receive(:normalize).with(value).and_return('normalized') + + hash['key'] = value + expect(hash['key']).to eql 'normalized' + end + + it 'can use #store as an alias to #[]=' do + hash.store 'key', 'value' + expect(hash['key']).to eql 'value' + end + + context 'with forbidden fields' do + let(:forbidden_keys) { ['forbidden', :foo, 42] } + + it 'denies setting a forbidden field' do + expect { hash[:forbidden] = 'value' }.to raise_error ArgumentError + expect { hash['forbidden'] = 'value' }.to raise_error ArgumentError + end + + it 'ignores non string-values in forbidden_keys' do + expect { hash[:foo] = 'value' }.not_to raise_error + expect { hash['foo'] = 'value' }.not_to raise_error + expect { hash[42] = 'value' }.not_to raise_error + expect { hash['42'] = 'value' }.not_to raise_error + expect { hash[:'42'] = 'value' }.not_to raise_error + end + + it 'returns nil when accessing forbidden fields' do + expect(hash['forbidden']).to be_nil + + expect(hash[:foo]).to be_nil + expect(hash['foo']).to be_nil + end + end + end + + describe '#as_json' do + before do + hash['simple'] = 'value' + hash['hash'] = { 'key' => 'nested value', number: 42 } + hash['array'] = ['v1', :v2] + end + + it 'returns a simple hash' do + expect(hash.as_json).to be_a ::Hash + expect(hash.as_json.keys).to eql %w[simple hash array] + end + + it 'returns a nested hash' do + expect(hash['hash']).to be_a Rackstash::Fields::Hash + + expect(hash.as_json['hash']).to be_a Hash + expect(hash.as_json['hash']).to eql 'key' => 'nested value', 'number' => 42 + end + + it 'returns a nested array' do + expect(hash['array']).to be_a Rackstash::Fields::Array + + expect(hash.as_json['array']).to be_an ::Array + expect(hash.as_json['array']).to eql %w[v1 v2] + end + + it 'returns a new copy each time' do + expect(hash.as_json).to eql hash.as_json + expect(hash.as_json).not_to equal hash.as_json + + expect(hash.as_json['hash']).to eql hash.as_json['hash'] + expect(hash.as_json['hash']).not_to equal hash.as_json['hash'] + + expect(hash.as_json['array']).to eql hash.as_json['array'] + expect(hash.as_json['array']).not_to equal hash.as_json['array'] + end + + it 'can not change the raw value' do + as_json = hash.as_json + as_json['injected'] = 'foo' + + expect(hash['injected']).to be_nil + expect(hash.keys).not_to include 'injected' + end + + it 'can use to_hash as an alias' do + expect(hash.to_hash).to eql hash.as_json + end + + it 'can use to_h as an alias' do + expect(hash.to_h).to eql hash.as_json + end + end + + describe '#clear' do + it 'clears the hash' do + hash['foo'] = 'bar' + hash.clear + expect(hash['foo']).to be_nil + expect(hash.keys).to be_empty + end + + it 'returns the hash' do + hash['foo'] = 'bar' + expect(hash.clear).to equal hash + end + end + + describe '#forbidden_key?' do + let(:forbidden_keys) { ['forbidden', :foo] } + + it 'checks if a key is forbidden' do + expect(hash.forbidden_key?('forbidden')).to be true + expect(hash.forbidden_key?('foo')).to be false + end + + end + + describe '#keys' do + it 'returns an array of keys' do + hash['foo'] = 'bar' + hash[:symbol] = 'symbol' + hash[42] = 'number' + + expect(hash.keys).to eql ['foo', 'symbol', '42'] + expect(hash.keys).to all be_frozen + end + + it 'returns a new array each time' do + expect(hash.keys).not_to equal hash.keys + end + end + + describe '#merge!' do + it 'rejects not hash-convertible arguments' do + expect { hash.merge!(nil) }.to raise_error TypeError + expect { hash.merge!(false) }.to raise_error TypeError + expect { hash.merge!(true) }.to raise_error TypeError + expect { hash.merge!(123) }.to raise_error TypeError + expect { hash.merge!(:foo) }.to raise_error TypeError + expect { hash.merge!('foo') }.to raise_error TypeError + expect { hash.merge!([]) }.to raise_error TypeError + expect { hash.merge!(['foo']) }.to raise_error TypeError + end + + it 'merges an empty hash with compatible arguments' do + empty_hash = Rackstash::Fields::Hash.new + + expect(hash.merge!({})).to eql empty_hash + expect(hash.merge!(Rackstash::Fields::Hash.new)).to eql empty_hash + end + + it 'merges a normalized hash' do + to_merge = {foo: :bar} + expect(hash).to receive(:normalize).with(to_merge, anything).ordered.and_call_original + expect(hash).to receive(:normalize).with(:bar, anything).ordered.and_call_original + + original_hash = hash + # the hash is mutated in place and returned + expect(hash.merge!(to_merge)).to equal original_hash + expect(hash['foo']).to eql 'bar' + expect(hash['foo']).to be_frozen + end + + it 'overwrites existing fields' do + hash['foo'] = 'bar' + + hash.merge!({ foo: 42 }, force: true) + expect(hash['foo']).to eql 42 + + hash.merge!({ foo: 'value' }, force: false) + expect(hash['foo']).to eql 'value' + end + + it 'calls the block on merge conflicts' do + hash['foo'] = 'bar' + + yielded_args = [] + yielded_count = 0 + + expect(hash).to receive(:normalize).with({ foo: 42 }, anything).ordered.and_call_original + expect(hash).to receive(:normalize).with(42, anything).ordered.and_call_original + expect(hash).to receive(:normalize).with(:symbol, anything).ordered.and_call_original + + hash.merge!(foo: 42) { |key, old_value, new_value| + yielded_count += 1 + yielded_args = [key, old_value, new_value] + :symbol + } + + expect(hash['foo']).to eql 'symbol' + expect(yielded_count).to eql 1 + expect(yielded_args).to eql ['foo', 'bar', 42] + end + + it 'resolves the value with the passed scope' do + scope = 'hello world' + + hash.merge!(-> { { key: self } }, scope: scope) + expect(hash['key']).to eql 'hello world' + + hash.merge!({ key: -> { { nested: self } } }, scope: scope) + expect(hash['key']['nested']).to eql 'hello world' + end + + context 'with forbidden_keys' do + let(:forbidden_keys) { ['forbidden'] } + + it 'raises an error when trying to merge forbidden_keys' do + expect { hash.merge!('forbidden' => 'v') }.to raise_error ArgumentError + expect { hash.merge!(forbidden: 'v') }.to raise_error ArgumentError + + expect { hash.merge!({ 'forbidden' => 'value' }, force: true) } + .to raise_error ArgumentError + expect { hash.merge!({ forbidden: 'value' }, force: true) } + .to raise_error ArgumentError + end + + it 'ignores forbidden_keys when not forcing' do + hash.merge!({ 'forbidden' => 'ignored' }, force: false) + expect(hash['forbidden']).to be_nil + end + end + end + + describe '#merge' do + it 'returns a new object' do + new_hash = hash.merge(foo: :bar) + + expect(new_hash).to be_a Rackstash::Fields::Hash + expect(new_hash).not_to equal hash + + # The origiginal hash is not changed + expect(hash['foo']).to be_nil + end + + describe 'with forbidden_keys' do + let(:forbidden_keys) { ['forbidden'] } + + it 'raises an error when trying to merge forbidden_keys' do + expect { hash.merge('forbidden' => 'v') }.to raise_error ArgumentError + end + + it 'ignores forbidden_keys when not forcing' do + new_hash = hash.merge({ 'forbidden' => 'ignored' }, force: false) + expect(new_hash['forbidden']).to be_nil + end + + it 'keeps the forbidden_keys on the new hash' do + new_hash = hash.merge({ 'forbidden' => 'ignored' }, force: false) + expect { new_hash.merge(forbidden: 'error') }.to raise_error ArgumentError + end + end + end + + describe '#values' do + it 'returns an array of values' do + hash['string'] = 'beep' + hash['float'] = 1.2 + hash['number'] = 42 + + expect(hash.values).to eql ['beep', 1.2, 42] + expect(hash.values).to all be_frozen + end + + it 'returns a new array each time' do + expect(hash.values).not_to equal hash.values + end + end + + describe 'Converter' do + it 'creates a new Hash' do + raw = { :time => Time.now, 'string' => 'foo' } + hash = Rackstash::Fields::Hash(raw) + + expect(hash).to be_a Rackstash::Fields::Hash + expect(hash['time']).to be_a String + expect(hash['string']).to eql 'foo' + end + + it 'can specify forbidden_keys' do + raw = { foo: :bar, forbidden: 'ignored' } + expect { Rackstash::Fields::Hash(raw, forbidden_fields: ['forbidden']) }.to raise_error ArgumentError + end + end +end diff --git a/spec/rackstash_spec.rb b/spec/rackstash_spec.rb index 534a8c0..8d8d94f 100644 --- a/spec/rackstash_spec.rb +++ b/spec/rackstash_spec.rb @@ -8,6 +8,7 @@ require 'spec_helper' describe Rackstash do it 'defines PROGRAME with the correct version' do expect(Rackstash::PROGNAME).to match %r{\Arackstash/v\d+(\..+)*\z} + expect(Rackstash::PROGNAME).to be_frozen end it 'defines SEVERITIES constants' do @@ -20,4 +21,14 @@ describe Rackstash do expect(Rackstash::FATAL).to eql 4 expect(Rackstash::UNKNOWN).to eql 5 end + + it 'defines EMPTY_* constants' do + expect(Rackstash::EMPTY_STRING).to eql '' + expect(Rackstash::EMPTY_STRING).to be_frozen + + expect(Rackstash::EMPTY_SET).to eql Set.new + expect(Rackstash::EMPTY_SET).to be_frozen + + expect(Rackstash::ISO8601_PRECISION).to be_a Integer + end end