Add basic fields to hold additional data on Buffers

The fields follow the basic structure of basic Hashes and Arrays but
provide an interface better suitable for us. Specifically:

* They check and enforce the datatypes for keys and values to be
  strictly JSON-conforming. Only the basic data-types are accepted
  respectively converted to.
* Hashes only accept String keys.
* Basic values are always frozen.

lib/rackstash.rb

@@ -3,6 +3,8 @@
 # This software may be modified and distributed under the terms
 # of the MIT license. See the LICENSE.txt file for details.
 
+require 'set'
+
 require 'rackstash/version'
 
 module Rackstash
@@ -18,6 +20,10 @@ module Rackstash
   PROGNAME = "rackstash/v#{Rackstash::VERSION}".freeze
 
   EMPTY_STRING = ''.freeze
+  EMPTY_SET = Set.new.freeze
+
+  # How many decimal places to render on ISO 8601 timestamps
+  ISO8601_PRECISION = 3
 end
 
 require 'rackstash/logger'

lib/rackstash/fields.rb

@@ -0,0 +1,8 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'rackstash/fields/abstract_collection'
+require 'rackstash/fields/hash'
+require 'rackstash/fields/array'

lib/rackstash/fields/abstract_collection.rb

@@ -0,0 +1,143 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'bigdecimal'
+require 'pathname'
+require 'uri'
+
+module Rackstash
+  module Fields
+    class AbstractCollection
+      def ==(other)
+        self.class == other.class && raw == other.raw
+      end
+      alias :eql? :==
+
+      def inspect
+        "#<#{self.class}:#{format '0x%014x', object_id << 1} #{to_s}>"
+      end
+
+      # Provide a copy of the wrapped {#raw} data in a format allowing direct
+      # mapping to JSON.
+      #
+      # @note This method is usually overridden in child classes.
+      def as_json(*)
+        nil
+      end
+
+      def to_json(*)
+        as_json.to_json
+      end
+
+      def to_s
+        as_json.inspect
+      end
+
+      protected
+
+      attr_accessor :raw
+
+      private
+
+      def initialize_copy(source)
+        super
+        self.raw = source.raw == nil ? nil : source.raw.dup
+        self
+      end
+
+      def new(raw)
+        self.class.new.tap do |new_object|
+          new_object.raw = raw
+        end
+      end
+
+      # @param str [#to_s]
+      def utf8_encode(str)
+        str.to_s.encode(
+          Encoding::UTF_8,
+          invalid: :replace,
+          undef: :replace
+        )
+      end
+
+      def resolve_value(value, scope: nil)
+        return value unless value.is_a?(Proc)
+        scope == nil ? value.call : scope.instance_exec(&value)
+      end
+
+      # Note: You should never mutate an array or hash returned by normalize
+      # when `wrap` is `false`.
+      def normalize(value, resolve: true, scope: nil, wrap: true)
+        value = resolve_value(value, scope: scope) if resolve
+
+        case value
+        when ::String, ::Symbol
+          return utf8_encode(value).freeze
+        when ::Integer, ::Float
+          return value
+        when true, false, nil
+          return value
+        when Rackstash::Fields::AbstractCollection
+          return wrap ? value : value.raw
+        when ::Hash
+          hash = value.each_with_object({}) do |(k, v), memo|
+            memo[utf8_encode(k)] = normalize(v, scope: scope, resolve: resolve)
+          end
+          hash = Rackstash::Fields::Hash.new.tap do |hash_field|
+            hash_field.raw = hash
+          end if wrap
+          return hash
+        when ::Array, ::Enumerator
+          array = value.map { |e| normalize(e, scope: scope, resolve: resolve) }
+          array = Rackstash::Fields::Array.new.tap do |array_field|
+            array_field.raw = array
+          end if wrap
+          return array
+        when ::Time
+          return value.utc.iso8601(ISO8601_PRECISION).freeze
+        when ::DateTime
+          return value.to_time.utc.iso8601(ISO8601_PRECISION).freeze
+        when ::Date
+          return value.iso8601.encode!(Encoding::UTF_8).freeze
+        when ::Regexp, ::Range, ::URI::Generic, ::Pathname
+          return utf8_encode(value).freeze
+        when Exception
+          exception = "#{value.message} (#{value.class})"
+          exception << "\n" << value.backtrace.join("\n") if value.backtrace
+          return utf8_encode(exception).freeze
+        when ::Proc
+          return resolve ? utf8_encode(value.inspect).freeze : value
+        when ::BigDecimal
+          # A BigDecimal would be naturally represented as a JSON number. Most
+          # libraries, however, parse non-integer JSON numbers directly as
+          # floats. Clients using those libraries would get in general a wrong
+          # number and no way to recover other than manually inspecting the
+          # string with the JSON code itself.
+          return value.to_s('F').encode!(Encoding::UTF_8).freeze
+        when ::Complex
+          # A complex number can not reliably converted to a float or rational,
+          # thus we always transform it to a String
+          return utf8_encode(value).freeze
+        end
+
+        # Try to convert the value to a known basic type and recurse
+        %i[
+          as_json
+          to_hash to_ary to_h to_a
+          to_time to_datetime to_date
+          to_f to_i
+        ].each do |method|
+          # Try to convert the value to a base type but ignore any errors
+          next unless value.respond_to?(method)
+          value = value.public_send(method) rescue next
+
+          return normalize(value, scope: scope, wrap: wrap, resolve: resolve)
+        end
+
+        utf8_encode(value.inspect).freeze
+      end
+    end
+  end
+end

lib/rackstash/fields/array.rb

@@ -0,0 +1,58 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'rackstash/fields/abstract_collection'
+
+module Rackstash
+  module Fields
+    class Array < AbstractCollection
+      def initialize
+        @raw = []
+      end
+
+      def [](index)
+        @raw[index]
+      end
+
+      def []=(index, value)
+        @raw[index] = normalize(value)
+      end
+
+      def as_json(*)
+        @raw.map { |value|
+          value.is_a?(AbstractCollection) ? value.as_json : value
+        }
+      end
+      alias :to_ary :as_json
+      alias :to_a :as_json
+
+      def clear
+        @raw.clear
+        self
+      end
+
+      def concat(array)
+        array = Array(normalize(array, wrap: false))
+        @raw.concat(array)
+        self
+      end
+
+      def length
+        @raw.length
+      end
+
+      private
+
+      def Array(obj)
+        return obj.to_ary if obj.respond_to?(:to_ary)
+        raise TypeError, "no implicit conversion of #{obj.class} into Array"
+      end
+    end
+
+    def self.Array(array)
+      Array.new.concat(array)
+    end
+  end
+end

lib/rackstash/fields/hash.rb

@@ -0,0 +1,100 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'rackstash/fields/abstract_collection'
+
+module Rackstash
+  module Fields
+    class Hash < AbstractCollection
+      def initialize(forbidden_keys: EMPTY_SET)
+        @raw = {}
+
+        if forbidden_keys.is_a?(Set)
+          forbidden_keys = forbidden_keys.dup.freeze unless forbidden_keys.frozen?
+          @forbidden_keys = forbidden_keys
+        else
+          @forbidden_keys = Set[*forbidden_keys].freeze
+        end
+      end
+
+      def [](key)
+        @raw[utf8_encode(key)]
+      end
+
+      def []=(key, value)
+        key = utf8_encode(key)
+        raise ArgumentError, "Forbidden field #{key}" if forbidden_key?(key)
+
+        @raw[key] = normalize(value)
+      end
+      alias :store :[]=
+
+      def as_json(*)
+        @raw.each_with_object({}) do |(key, value), memo|
+          value = value.as_json if value.is_a?(AbstractCollection)
+          memo[key] = value
+        end
+      end
+      alias :to_hash :as_json
+      alias :to_h :as_json
+
+      def clear
+        @raw.clear
+        self
+      end
+
+      def keys
+        @raw.keys
+      end
+
+      def merge(hash, force: true, scope: nil, &block)
+        dup.merge!(hash, force: force, scope: scope, &block)
+      end
+
+      def merge!(hash, force: true, scope: nil)
+        hash = Hash(normalize(hash, scope: scope, wrap: false))
+
+        if force
+          forbidden = @forbidden_keys & hash.keys
+          unless forbidden.empty?
+            raise ArgumentError, "Forbidden keys #{forbidden.to_a.join(', ')}"
+          end
+        else
+          hash = hash.reject { |k, _v| forbidden_key?(k) }
+        end
+
+        if block_given?
+          @raw.merge!(hash) { |key, old_val, new_val|
+            yielded = yield(key, old_val, new_val)
+            normalize(yielded, scope: scope)
+          }
+        else
+          @raw.merge!(hash)
+        end
+        self
+      end
+      alias :update :merge!
+
+      def forbidden_key?(key)
+        @forbidden_keys.include?(key)
+      end
+
+      def values
+        @raw.values
+      end
+
+      private
+
+      def Hash(obj)
+        return obj.to_hash if obj.respond_to?(:to_hash)
+        raise TypeError, "no implicit conversion of #{obj.class} into Hash"
+      end
+    end
+
+    def self.Hash(raw, forbidden_keys: EMPTY_SET)
+      Hash.new(forbidden_keys: forbidden_keys).merge!(raw)
+    end
+  end
+end

spec/rackstash/fields/abstract_collection_spec.rb

@@ -0,0 +1,523 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'spec_helper'
+
+require 'rackstash/fields/abstract_collection'
+require 'rackstash/fields/array'
+require 'rackstash/fields/hash'
+
+describe Rackstash::Fields::AbstractCollection do
+  let(:collection) { Rackstash::Fields::AbstractCollection.new }
+
+  def normalize(*args)
+    collection.send(:normalize, *args)
+  end
+
+  describe '#to_json' do
+    it 'returns the JSON version of as_json' do
+      as_json = double('JSON value')
+      expect(collection).to receive(:as_json).and_return(as_json)
+      expect(as_json).to receive(:to_json)
+
+      collection.to_json
+    end
+  end
+
+  describe '#inspect' do
+    it 'formats the object' do
+      expect(collection).to receive(:as_json).and_return('beepboop')
+      expect(collection.inspect).to(
+        match %r{\A#<Rackstash::Fields::AbstractCollection:0x[a-f0-9]+ "beepboop">\z}
+      )
+    end
+  end
+
+  describe '#eql?' do
+    it 'is equal with the same class with the same raw value' do
+      expect(collection).to receive(:eql?).twice.and_call_original
+      expect(collection).to receive(:==).twice.and_call_original
+
+      other = Rackstash::Fields::AbstractCollection.new
+      expect(collection).to eql other
+      expect(collection).to eq other
+
+      other.send(:raw=, 'different value')
+      expect(collection).not_to eql other
+      expect(collection).not_to eq other
+    end
+
+    it 'is not equal on different classes' do
+      other = Struct.new(:raw).new
+      expect(collection.send(:raw)).to eql other.raw
+
+      expect(collection).to receive(:eql?).and_call_original
+      expect(collection).not_to eql other
+
+      expect(collection).to receive(:==).and_call_original
+      expect(collection).not_to eq other
+    end
+  end
+
+  describe '#dup' do
+    it 'dups the raw value' do
+      raw = 'hello'
+      collection.send(:raw=, raw)
+      expect(collection.send(:raw)).to equal raw
+
+      duped = collection.dup
+
+      expect(duped).not_to equal collection
+      expect(duped.send(:raw)).to eql 'hello'
+      expect(duped.send(:raw)).not_to equal raw
+    end
+  end
+
+  describe '#raw' do
+    it 'is a protected accessor' do
+      expect { collection.raw = nil }.to raise_error NoMethodError
+      expect { collection.raw }.to raise_error NoMethodError
+
+      collection.send(:raw=, 'beep')
+      expect(collection.send(:raw)).to eql 'beep'
+    end
+  end
+
+  describe '#normalize' do
+    describe 'with String' do
+      it 'transforms encoding to UTF-8' do
+        utf8_str = 'Dönerstraße'
+        latin_str = utf8_str.encode(Encoding::ISO8859_9)
+        expect(latin_str.encoding).to eql Encoding::ISO8859_9
+
+        expect(normalize(latin_str)).to eql utf8_str
+        expect(normalize(latin_str).encoding).to eql Encoding::UTF_8
+        expect(normalize(latin_str)).to be_frozen
+      end
+
+      it 'replaces invalid characters in correctly encoded strings' do
+        binary = Digest::SHA256.digest('string')
+
+        expect(normalize(binary)).to include '<27>'
+        expect(normalize(binary).encoding).to eql Encoding::UTF_8
+        expect(normalize(binary)).to be_frozen
+      end
+
+      it 'replaces invalid characters in incorrectly encoded strings' do
+        strange = Digest::SHA256.digest('string').force_encoding(Encoding::UTF_8)
+
+        expect(normalize(strange)).to include '<27>'
+        expect(normalize(strange).encoding).to eql Encoding::UTF_8
+        expect(normalize(strange)).to be_frozen
+      end
+
+      it 'does not alter valid strings' do
+        valid = 'Dönerstraße'.freeze
+
+        expect(normalize(valid)).to eql valid
+        # All strings are still copied and frozen, even if they don't need to
+        # be re-encoded.
+        expect(normalize(valid)).not_to equal valid
+        expect(normalize(valid)).to be_frozen
+      end
+    end
+
+    it 'transforms Symbol to String' do
+      symbol = :foo
+
+      expect(normalize(symbol)).to eql 'foo'
+      expect(normalize(symbol).encoding).to eql Encoding::UTF_8
+      expect(normalize(symbol)).to be_frozen
+    end
+
+    it 'passes Integer' do
+      fixnum = 42
+      expect(normalize(fixnum)).to equal fixnum
+      expect(normalize(fixnum)).to be_frozen
+
+      bignum = 10**100
+      expect(normalize(bignum)).to equal bignum
+      expect(normalize(bignum)).to be_frozen
+    end
+
+    it 'passes Float' do
+      float = 123.456
+
+      expect(normalize(float)).to equal float
+      expect(normalize(float)).to be_frozen
+    end
+
+    it 'passes true, false, nil' do
+      expect(normalize(true)).to equal true
+      expect(normalize(false)).to equal false
+      expect(normalize(nil)).to equal nil
+    end
+
+    describe 'with Rackstash::Fields::AbstractCollection' do
+      let(:raw) { 'beepboop' }
+      let(:value) {
+        value = double('Rackstash::Fields::AbstractCollection')
+        allow(value).to receive(:raw).and_return raw
+        value
+      }
+
+      before do
+        expect(Rackstash::Fields::AbstractCollection).to(
+          receive(:===).with(value).and_return(true)
+        )
+      end
+
+      it 'passes the collection by default' do
+        expect(normalize(value)).to equal value
+      end
+
+      it 'unwraps the collection if selected' do
+        expect(normalize(value, wrap: false)).to equal raw
+      end
+    end
+
+    describe 'with Hash' do
+      it 'wraps the hash in a Rackstash::Fields::Hash' do
+        hash = { 'beep' => 'boop' }
+
+        expect(normalize(hash)).to be_a Rackstash::Fields::Hash
+        expect(normalize(hash).send(:raw)).to eql 'beep' => 'boop'
+        expect(normalize(hash).send(:raw)).to_not equal hash
+      end
+
+      it 'normalizes keys to frozen UTF-8 strings' do
+        hash = { 1 => 1, :two => 2, 'three' => 3, nil => 4 }
+
+        expect(normalize(hash, wrap: false)).to eql(
+          { '1' => 1, 'two' => 2, 'three' => 3, '' => 4 }
+        )
+        expect(normalize(hash, wrap: false).keys).to all be_frozen
+      end
+
+      it 'normalizes all values' do
+        hash = { 'key' => :beepboop }
+
+        expect(collection).to receive(:normalize).with(hash).ordered
+          .twice.and_call_original
+        expect(collection).to receive(:normalize).with(:beepboop, anything).ordered
+          .twice.and_call_original
+
+        expect(normalize(hash)).to be_a Rackstash::Fields::Hash
+        expect(normalize(hash).send(:raw)).to eql 'key' => 'beepboop'
+      end
+
+      it 'deep-wraps the hash' do
+        hash = { beep: { rawr: 'growl' } }
+
+        expect(normalize(hash)).to be_a Rackstash::Fields::Hash
+        expect(normalize(hash)['beep']).to be_a Rackstash::Fields::Hash
+        expect(normalize(hash)['beep']['rawr']).to eql 'growl'
+      end
+
+      it 'copies the hash' do
+        raw_hash = { beep: 'boing' }
+        wrapped_hash = normalize(raw_hash)
+
+        raw_hash['foo'] = 'bar'
+        expect(wrapped_hash['foo']).to be_nil
+      end
+
+      describe 'with procs' do
+        it 'resolves values' do
+          hash = { beep: -> { { rawr: -> { 'growl' } } } }
+
+          expect(normalize(hash)).to be_a Rackstash::Fields::Hash
+          expect(normalize(hash)['beep']).to be_a Rackstash::Fields::Hash
+          expect(normalize(hash)['beep']['rawr']).to eql 'growl'
+        end
+
+        it 'resolves values with the supplied scope' do
+          scope = 'scope'
+          hash = { beep: -> { { self => -> { upcase } } } }
+
+          expect(normalize(hash, scope: scope)).to be_a Rackstash::Fields::Hash
+          expect(normalize(hash, scope: scope)['beep']).to be_a Rackstash::Fields::Hash
+          expect(normalize(hash, scope: scope)['beep']['scope']).to eql 'SCOPE'
+        end
+      end
+    end
+
+    describe 'with Array' do
+      it 'wraps the array in a Rackstash::Fields::Array' do
+        array = ['beep', 'boop']
+
+        expect(normalize(array)).to be_a Rackstash::Fields::Array
+        expect(normalize(array).send(:raw)).to eql ['beep', 'boop']
+        expect(normalize(array).send(:raw)).to_not equal array
+      end
+
+      it 'normalizes values to frozen UTF-8 strings' do
+        array = [1, :two, 'three', nil]
+
+        expect(normalize(array, wrap: false)).to eql [1, 'two', 'three', nil]
+        expect(normalize(array, wrap: false)).to all be_frozen
+      end
+
+      it 'normalizes all values' do
+        array = ['boop', :beep]
+
+        expect(collection).to receive(:normalize).with(array).ordered
+          .twice.and_call_original
+        expect(collection).to receive(:normalize).with('boop', anything).ordered
+          .twice.and_call_original
+        expect(collection).to receive(:normalize).with(:beep, anything).ordered
+          .twice.and_call_original
+
+        expect(normalize(array)).to be_a Rackstash::Fields::Array
+        expect(normalize(array).send(:raw)).to eql ['boop', 'beep']
+      end
+
+      it 'deep-wraps the array' do
+        array = [123, ['foo', :bar]]
+
+        expect(normalize(array)).to be_a Rackstash::Fields::Array
+        expect(normalize(array)[0]).to eql 123
+        expect(normalize(array)[1]).to be_a Rackstash::Fields::Array
+        expect(normalize(array)[1][0]).to eql 'foo'
+        expect(normalize(array)[1][1]).to eql 'bar'
+      end
+
+      it 'copies the array' do
+        raw_array = [12, 'boing']
+        wrapped_array = normalize(raw_array)
+
+        raw_array[2] = 'foo'
+        expect(wrapped_array[2]).to be_nil
+      end
+
+      describe 'with procs' do
+        it 'resolves values' do
+          array = [123, -> { ['foo', -> { :bar }] }]
+
+          expect(normalize(array)).to be_a Rackstash::Fields::Array
+          expect(normalize(array)[1]).to be_a Rackstash::Fields::Array
+          expect(normalize(array)[1][1]).to eql 'bar'
+        end
+
+        it 'resolves values with the supplied scope' do
+          scope = 'string'.freeze
+          array = [123, -> { [upcase, -> { self }] }]
+
+          expect(normalize(array, scope: scope)[1][0]).to eql 'STRING'
+          expect(normalize(array, scope: scope)[1][1]).to eql scope
+        end
+      end
+    end
+
+    it 'wraps an Enumerator in a Rackstash::Fields::Array' do
+      small_numbers = Enumerator.new do |y|
+        3.times do |i|
+          y << i
+        end
+      end
+      expect(normalize(small_numbers)).to be_a Rackstash::Fields::Array
+      expect(normalize(small_numbers).send(:raw)).to eql [0, 1, 2]
+    end
+
+    it 'formats Time as an ISO 8601 UTC timestamp' do
+      time = Time.parse('2016-10-17 15:37:42 CEST') # UTC +02:00
+
+      expect(normalize(time)).to eql '2016-10-17T13:37:42.000Z'
+      expect(normalize(time)).to be_frozen
+    end
+
+    it 'formats Time as an ISO 8601 UTC timestamp' do
+      time = Time.parse('2016-10-17 15:37:42 CEST') # UTC +02:00
+
+      expect(normalize(time)).to eql '2016-10-17T13:37:42.000Z'
+      expect(normalize(time).encoding).to eql Encoding::UTF_8
+      expect(normalize(time)).to be_frozen
+    end
+
+    it 'formats DateTime as an ISO 8601 UTC timestamp' do
+      datetime = DateTime.parse('2016-10-17 15:37:42 CEST') # UTC +02:00
+
+      expect(normalize(datetime)).to eql '2016-10-17T13:37:42.000Z'
+      expect(normalize(datetime).encoding).to eql Encoding::UTF_8
+      expect(normalize(datetime)).to be_frozen
+    end
+
+    it 'formats Date as an ISO 8601 date string' do
+      date = Date.new(2016, 10, 17)
+
+      expect(normalize(date)).to eql '2016-10-17'
+      expect(normalize(date).encoding).to eql Encoding::UTF_8
+      expect(normalize(date)).to be_frozen
+    end
+
+    it 'transforms Regexp to String' do
+      regexp = /.?|(..+?)\1+/
+
+      expect(normalize(regexp)).to eql '(?-mix:.?|(..+?)\1+)'
+      expect(normalize(regexp).encoding).to eql Encoding::UTF_8
+      expect(normalize(regexp)).to be_frozen
+    end
+
+    it 'transforms Range to String' do
+      range = (1..10)
+
+      expect(normalize(range)).to eql '1..10'
+      expect(normalize(range).encoding).to eql Encoding::UTF_8
+      expect(normalize(range)).to be_frozen
+    end
+
+    it 'transforms URI to String' do
+      uris = {
+        URI('https://example.com/p/f.txt') => 'https://example.com/p/f.txt',
+        URI('') => ''
+      }
+
+      uris.each do |uri, result|
+        expect(uri).to be_a URI::Generic
+
+        expect(normalize(uri)).to eql result
+        expect(normalize(uri).encoding).to eql Encoding::UTF_8
+        expect(normalize(uri)).to be_frozen
+      end
+    end
+
+    it 'transforms URI to String' do
+      uris = {
+        URI('https://example.com/p/f.txt') => 'https://example.com/p/f.txt',
+        URI('') => ''
+      }
+
+      uris.each do |uri, result|
+        expect(uri).to be_a URI::Generic
+
+        expect(normalize(uri)).to eql result
+        expect(normalize(uri).encoding).to eql Encoding::UTF_8
+        expect(normalize(uri)).to be_frozen
+      end
+    end
+
+    it 'transforms Pathname to String' do
+      pathname = Pathname.new('/path/to/file.ext'.encode(Encoding::ISO8859_9))
+
+      expect(normalize(pathname)).to eql '/path/to/file.ext'
+      expect(normalize(pathname).encoding).to eql Encoding::UTF_8
+      expect(normalize(pathname)).to be_frozen
+    end
+
+    it 'formats an Exception with Backtrace' do
+      exception = nil
+      begin
+        raise StandardError, 'An Error'
+      rescue => e
+        exception = e
+      end
+
+      checker = Regexp.new <<-EOF.gsub(/^\s+/, '').rstrip, Regexp::MULTILINE
+        \\AAn Error \\(StandardError\\)
+        #{Regexp.escape __FILE__}:#{__LINE__ - 7}:in `block .*`
+      EOF
+      expect(normalize(exception)).to match checker
+      expect(normalize(exception).encoding).to eql Encoding::UTF_8
+      expect(normalize(exception)).to be_frozen
+    end
+
+    it 'transforms BigDecimal to String' do
+      bigdecimal = BigDecimal.new('123.987653')
+
+      expect(normalize(bigdecimal)).to eql '123.987653'
+      expect(normalize(bigdecimal).encoding).to eql Encoding::UTF_8
+      expect(normalize(bigdecimal)).to be_frozen
+    end
+
+    describe 'with Proc' do
+      it 'calls the proc by default and normalizes the result' do
+        proc = proc { :return }
+
+        expect(normalize(proc)).to eql 'return'
+        expect(normalize(proc).encoding).to eql Encoding::UTF_8
+        expect(normalize(proc)).to be_frozen
+      end
+
+      it 'inspects a nested proc' do
+        inner = proc { :return }
+        outer = proc { inner }
+
+        expect(normalize(outer)).to match %r{\A#<Proc:0x[0-9a-f]+@#{__FILE__}:#{__LINE__-3}>\z}
+      end
+
+      it 'returns the proc when not resolving' do
+        outer = proc { :return }
+
+        expect(normalize(outer, resolve: false)).to equal outer
+      end
+    end
+
+    it 'transforms Complex to String' do
+      complex = Complex(2, 3)
+
+      expect(normalize(complex)).to eql '2+3i'
+      expect(normalize(complex).encoding).to eql Encoding::UTF_8
+      expect(normalize(complex)).to be_frozen
+    end
+
+    it 'transforms Rational to Float' do
+      rational = Rational(-8, 6)
+
+      expect(normalize(rational)).to be_a Float
+      expect(normalize(rational)).to be_frozen
+    end
+
+    context 'conversion methods' do
+      let(:methods) {
+        %i[as_json to_hash to_ary to_h to_a to_time to_datetime to_date to_f to_i]
+      }
+
+      it 'attempts conversion to base objects in order' do
+        methods.each_with_index do |method, i|
+          obj = double("#{method} - successful")
+
+          methods[0..i].each_with_index do |check, j|
+            expect(obj).to receive(:respond_to?).with(check).and_return(i==j)
+              .ordered.once
+          end
+
+          expect(obj).to receive(method).and_return("obj with #{method}")
+            .ordered.once
+          expect(normalize(obj)).to eql "obj with #{method}"
+        end
+      end
+
+      it 'falls back on conversion error' do
+        obj = double('erroneous')
+
+        methods.each do |method|
+          expect(obj).to receive(:respond_to?).with(method).and_return(true)
+            .ordered.once
+          expect(obj).to receive(method).and_raise('foo').ordered.once
+        end
+
+        expect(obj).to receive(:inspect).and_return 'finally'
+        expect(normalize(obj)).to eql 'finally'
+      end
+
+      it 'inspects objects we don\'t have a special rule for' do
+        obj = double('any object')
+        expect(obj).to receive(:inspect).and_return('an object')
+
+        expect(normalize(obj)).to eql 'an object'
+      end
+    end
+  end
+
+  describe '#to_s' do
+    it 'inspects #as_json' do
+      as_json = double('JSON value')
+      expect(collection).to receive(:as_json).and_return(as_json)
+      expect(as_json).to receive(:inspect)
+
+      collection.to_s
+    end
+  end
+end

spec/rackstash/fields/array_spec.rb

@@ -0,0 +1,148 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'spec_helper'
+
+require 'rackstash/fields/array'
+
+describe Rackstash::Fields::Array do
+  let(:array) { Rackstash::Fields::Array.new }
+
+  describe '#[]' do
+    it 'returns nil if a value was not set' do
+      expect(array[1]).to be_nil
+    end
+
+    it 'returns a set value' do
+      array[0] = 'value'
+      expect(array[0]).to eql 'value'
+    end
+  end
+
+  describe '#[]=' do
+    it 'normalizes values' do
+      expect(array).to receive(:normalize).with('value').and_return('normalized')
+
+      array[0] = 'value'
+      expect(array[0]).to eql 'normalized'
+    end
+  end
+
+  describe '#as_json' do
+    before do
+      array[0] = 'value'
+      array[1] = { 'key' => 'nested value', number: 42 }
+      array[2] = ['v1', :v2]
+    end
+
+    it 'returns a simple array' do
+      expect(array.as_json).to be_a ::Array
+      expect(array.as_json.length).to eql 3
+    end
+
+    it 'returns a nested hash' do
+      expect(array[1]).to be_a Rackstash::Fields::Hash
+
+      expect(array.as_json[1]).to be_a Hash
+      expect(array.as_json[1]).to eql 'key' => 'nested value', 'number' => 42
+    end
+
+    it 'returns a nested array' do
+      expect(array[2]).to be_a Rackstash::Fields::Array
+
+      expect(array.as_json[2]).to be_an ::Array
+      expect(array.as_json[2]).to eql %w[v1 v2]
+    end
+
+    it 'returns a new copy each time' do
+      expect(array.as_json).to eql array.as_json
+      expect(array.as_json).not_to equal array.as_json
+
+      expect(array.as_json[1]).to eql array.as_json[1]
+      expect(array.as_json[1]).not_to equal array.as_json[1]
+
+      expect(array.as_json[2]).to eql array.as_json[2]
+      expect(array.as_json[2]).not_to equal array.as_json[2]
+    end
+
+    it 'can not change the raw value' do
+      as_json = array.as_json
+      as_json[3] = 'foo'
+
+      expect(array[3]).to be_nil
+    end
+
+    it 'can use to_ary as an alias' do
+      expect(array.to_ary).to eql array.as_json
+    end
+
+    it 'can use to_a as an alias' do
+      expect(array.to_a).to eql array.as_json
+    end
+  end
+
+  describe '#clear' do
+    it 'clears the array' do
+      array[0] = 'beep'
+      array.clear
+      expect(array[0]).to be_nil
+    end
+
+    it 'returns the array' do
+      array[0] = 'bar'
+      expect(array.clear).to equal array
+    end
+  end
+
+  describe '#concat' do
+    it 'contacts an array' do
+      array[0] = 'first'
+      ary = ['foo', 'bar']
+
+      expect(array).to receive(:normalize).with(ary, anything).ordered.and_call_original
+      expect(array).to receive(:normalize).with('foo', anything).ordered.and_call_original
+      expect(array).to receive(:normalize).with('bar', anything).ordered.and_call_original
+
+      expect(array.concat(ary)).to equal array
+
+      expect(array[0]).to eql 'first'
+
+      expect(array[1]).to eql 'foo'
+      expect(array[1]).to be_frozen
+      expect(array[2]).to eql 'bar'
+      expect(array[2]).to be_frozen
+    end
+
+    it 'refuses to concat an arbitrary value' do
+      expect { array.concat(:foo) }.to raise_error TypeError
+      expect { array.concat(42) }.to raise_error TypeError
+      expect { array.concat(false) }.to raise_error TypeError
+      expect { array.concat(nil) }.to raise_error TypeError
+    end
+  end
+
+  describe '#length' do
+    it 'returns the length of the array' do
+      expect(array.length).to eql 0
+
+      array[0] = 'first'
+      expect(array.length).to eql 1
+
+      array.clear
+      expect(array.length).to eql 0
+    end
+  end
+
+  describe 'Converter' do
+    it 'creates a new array' do
+      raw = [Time.now, 'foo']
+      array = Rackstash::Fields::Array(raw)
+
+      expect(array).to be_a Rackstash::Fields::Array
+      expect(array[0]).to be_a String
+      expect(array[1]).to eql 'foo'
+    end
+  end
+end

spec/rackstash/fields/hash_spec.rb

@@ -0,0 +1,340 @@
+# Copyright 2017 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'spec_helper'
+
+require 'rackstash/fields/hash'
+
+describe Rackstash::Fields::Hash do
+  let(:forbidden_keys) { Set.new }
+  let(:hash) { Rackstash::Fields::Hash.new(forbidden_keys: forbidden_keys) }
+
+  describe '#initialize' do
+    it 'can be initialized without any arguments' do
+      Rackstash::Fields::Hash.new
+    end
+
+    it 'accepts forbidden_keys as an Array' do
+      hash = Rackstash::Fields::Hash.new(forbidden_keys: ['field'])
+      expect(hash.instance_variable_get('@forbidden_keys')).to be_a Set
+    end
+
+    it 'accepts forbidden_keys as a Set' do
+      hash = Rackstash::Fields::Hash.new(forbidden_keys: Set['field'])
+      expect(hash.instance_variable_get('@forbidden_keys')).to be_a Set
+    end
+  end
+
+  describe 'subscript accessors' do
+    it 'normalizes keys when setting values' do
+      hash[:foo] = 'foo value'
+      expect(hash['foo']).to eql 'foo value'
+
+      hash[42] = '42 value'
+      expect(hash['42']).to eql '42 value'
+
+      hash[nil] = 'nil value'
+      expect(hash['']).to eql 'nil value'
+    end
+
+    it 'normalizes keys when accessing values' do
+      hash['foo'] = 'foo value'
+      expect(hash[:foo]).to eql 'foo value'
+
+      hash['42'] = '42 value'
+      expect(hash[42]).to eql '42 value'
+
+      hash[''] = 'nil value'
+      expect(hash[nil]).to eql 'nil value'
+    end
+
+    it 'returns nil if a value was not set' do
+      expect(hash['missing']).to be_nil
+    end
+
+    it 'normalizes values' do
+      value = 'value'
+      expect(hash).to receive(:normalize).with(value).and_return('normalized')
+
+      hash['key'] = value
+      expect(hash['key']).to eql 'normalized'
+    end
+
+    it 'can use #store as an alias to #[]=' do
+      hash.store 'key', 'value'
+      expect(hash['key']).to eql 'value'
+    end
+
+    context 'with forbidden fields' do
+      let(:forbidden_keys) { ['forbidden', :foo, 42] }
+
+      it 'denies setting a forbidden field' do
+        expect { hash[:forbidden] = 'value' }.to raise_error ArgumentError
+        expect { hash['forbidden'] = 'value' }.to raise_error ArgumentError
+      end
+
+      it 'ignores non string-values in forbidden_keys' do
+        expect { hash[:foo] = 'value' }.not_to raise_error
+        expect { hash['foo'] = 'value' }.not_to raise_error
+        expect { hash[42] = 'value' }.not_to raise_error
+        expect { hash['42'] = 'value' }.not_to raise_error
+        expect { hash[:'42'] = 'value' }.not_to raise_error
+      end
+
+      it 'returns nil when accessing forbidden fields' do
+        expect(hash['forbidden']).to be_nil
+
+        expect(hash[:foo]).to be_nil
+        expect(hash['foo']).to be_nil
+      end
+    end
+  end
+
+  describe '#as_json' do
+    before do
+      hash['simple'] = 'value'
+      hash['hash'] = { 'key' => 'nested value', number: 42 }
+      hash['array'] = ['v1', :v2]
+    end
+
+    it 'returns a simple hash' do
+      expect(hash.as_json).to be_a ::Hash
+      expect(hash.as_json.keys).to eql %w[simple hash array]
+    end
+
+    it 'returns a nested hash' do
+      expect(hash['hash']).to be_a Rackstash::Fields::Hash
+
+      expect(hash.as_json['hash']).to be_a Hash
+      expect(hash.as_json['hash']).to eql 'key' => 'nested value', 'number' => 42
+    end
+
+    it 'returns a nested array' do
+      expect(hash['array']).to be_a Rackstash::Fields::Array
+
+      expect(hash.as_json['array']).to be_an ::Array
+      expect(hash.as_json['array']).to eql %w[v1 v2]
+    end
+
+    it 'returns a new copy each time' do
+      expect(hash.as_json).to eql hash.as_json
+      expect(hash.as_json).not_to equal hash.as_json
+
+      expect(hash.as_json['hash']).to eql hash.as_json['hash']
+      expect(hash.as_json['hash']).not_to equal hash.as_json['hash']
+
+      expect(hash.as_json['array']).to eql hash.as_json['array']
+      expect(hash.as_json['array']).not_to equal hash.as_json['array']
+    end
+
+    it 'can not change the raw value' do
+      as_json = hash.as_json
+      as_json['injected'] = 'foo'
+
+      expect(hash['injected']).to be_nil
+      expect(hash.keys).not_to include 'injected'
+    end
+
+    it 'can use to_hash as an alias' do
+      expect(hash.to_hash).to eql hash.as_json
+    end
+
+    it 'can use to_h as an alias' do
+      expect(hash.to_h).to eql hash.as_json
+    end
+  end
+
+  describe '#clear' do
+    it 'clears the hash' do
+      hash['foo'] = 'bar'
+      hash.clear
+      expect(hash['foo']).to be_nil
+      expect(hash.keys).to be_empty
+    end
+
+    it 'returns the hash' do
+      hash['foo'] = 'bar'
+      expect(hash.clear).to equal hash
+    end
+  end
+
+  describe '#forbidden_key?' do
+    let(:forbidden_keys) { ['forbidden', :foo] }
+
+    it 'checks if a key is forbidden' do
+      expect(hash.forbidden_key?('forbidden')).to be true
+      expect(hash.forbidden_key?('foo')).to be false
+    end
+
+  end
+
+  describe '#keys' do
+    it 'returns an array of keys' do
+      hash['foo'] = 'bar'
+      hash[:symbol] = 'symbol'
+      hash[42] = 'number'
+
+      expect(hash.keys).to eql ['foo', 'symbol', '42']
+      expect(hash.keys).to all be_frozen
+    end
+
+    it 'returns a new array each time' do
+      expect(hash.keys).not_to equal hash.keys
+    end
+  end
+
+  describe '#merge!' do
+    it 'rejects not hash-convertible arguments' do
+      expect { hash.merge!(nil) }.to raise_error TypeError
+      expect { hash.merge!(false) }.to raise_error TypeError
+      expect { hash.merge!(true) }.to raise_error TypeError
+      expect { hash.merge!(123) }.to raise_error TypeError
+      expect { hash.merge!(:foo) }.to raise_error TypeError
+      expect { hash.merge!('foo') }.to raise_error TypeError
+      expect { hash.merge!([]) }.to raise_error TypeError
+      expect { hash.merge!(['foo']) }.to raise_error TypeError
+    end
+
+    it 'merges an empty hash with compatible arguments' do
+      empty_hash = Rackstash::Fields::Hash.new
+
+      expect(hash.merge!({})).to eql empty_hash
+      expect(hash.merge!(Rackstash::Fields::Hash.new)).to eql empty_hash
+    end
+
+    it 'merges a normalized hash' do
+      to_merge = {foo: :bar}
+      expect(hash).to receive(:normalize).with(to_merge, anything).ordered.and_call_original
+      expect(hash).to receive(:normalize).with(:bar, anything).ordered.and_call_original
+
+      original_hash = hash
+      # the hash is mutated in place and returned
+      expect(hash.merge!(to_merge)).to equal original_hash
+      expect(hash['foo']).to eql 'bar'
+      expect(hash['foo']).to be_frozen
+    end
+
+    it 'overwrites existing fields' do
+      hash['foo'] = 'bar'
+
+      hash.merge!({ foo: 42 }, force: true)
+      expect(hash['foo']).to eql 42
+
+      hash.merge!({ foo: 'value' }, force: false)
+      expect(hash['foo']).to eql 'value'
+    end
+
+    it 'calls the block on merge conflicts' do
+      hash['foo'] = 'bar'
+
+      yielded_args = []
+      yielded_count = 0
+
+      expect(hash).to receive(:normalize).with({ foo: 42 }, anything).ordered.and_call_original
+      expect(hash).to receive(:normalize).with(42, anything).ordered.and_call_original
+      expect(hash).to receive(:normalize).with(:symbol, anything).ordered.and_call_original
+
+      hash.merge!(foo: 42) { |key, old_value, new_value|
+        yielded_count += 1
+        yielded_args = [key, old_value, new_value]
+        :symbol
+      }
+
+      expect(hash['foo']).to eql 'symbol'
+      expect(yielded_count).to eql 1
+      expect(yielded_args).to eql ['foo', 'bar', 42]
+    end
+
+    it 'resolves the value with the passed scope' do
+      scope = 'hello world'
+
+      hash.merge!(-> { { key: self } }, scope: scope)
+      expect(hash['key']).to eql 'hello world'
+
+      hash.merge!({ key: -> { { nested: self } } }, scope: scope)
+      expect(hash['key']['nested']).to eql 'hello world'
+    end
+
+    context 'with forbidden_keys' do
+      let(:forbidden_keys) { ['forbidden'] }
+
+      it 'raises an error when trying to merge forbidden_keys' do
+        expect { hash.merge!('forbidden' => 'v') }.to raise_error ArgumentError
+        expect { hash.merge!(forbidden: 'v') }.to raise_error ArgumentError
+
+        expect { hash.merge!({ 'forbidden' => 'value' }, force: true) }
+          .to raise_error ArgumentError
+        expect { hash.merge!({ forbidden: 'value' }, force: true) }
+          .to raise_error ArgumentError
+      end
+
+      it 'ignores forbidden_keys when not forcing' do
+        hash.merge!({ 'forbidden' => 'ignored' }, force: false)
+        expect(hash['forbidden']).to be_nil
+      end
+    end
+  end
+
+  describe '#merge' do
+    it 'returns a new object' do
+      new_hash = hash.merge(foo: :bar)
+
+      expect(new_hash).to be_a Rackstash::Fields::Hash
+      expect(new_hash).not_to equal hash
+
+      # The origiginal hash is not changed
+      expect(hash['foo']).to be_nil
+    end
+
+    describe 'with forbidden_keys' do
+      let(:forbidden_keys) { ['forbidden'] }
+
+      it 'raises an error when trying to merge forbidden_keys' do
+        expect { hash.merge('forbidden' => 'v') }.to raise_error ArgumentError
+      end
+
+      it 'ignores forbidden_keys when not forcing' do
+        new_hash = hash.merge({ 'forbidden' => 'ignored' }, force: false)
+        expect(new_hash['forbidden']).to be_nil
+      end
+
+      it 'keeps the forbidden_keys on the new hash' do
+        new_hash = hash.merge({ 'forbidden' => 'ignored' }, force: false)
+        expect { new_hash.merge(forbidden: 'error') }.to raise_error ArgumentError
+      end
+    end
+  end
+
+  describe '#values' do
+    it 'returns an array of values' do
+      hash['string'] = 'beep'
+      hash['float'] = 1.2
+      hash['number'] = 42
+
+      expect(hash.values).to eql ['beep', 1.2, 42]
+      expect(hash.values).to all be_frozen
+    end
+
+    it 'returns a new array each time' do
+      expect(hash.values).not_to equal hash.values
+    end
+  end
+
+  describe 'Converter' do
+    it 'creates a new Hash' do
+      raw = { :time => Time.now, 'string' => 'foo' }
+      hash = Rackstash::Fields::Hash(raw)
+
+      expect(hash).to be_a Rackstash::Fields::Hash
+      expect(hash['time']).to be_a String
+      expect(hash['string']).to eql 'foo'
+    end
+
+    it 'can specify forbidden_keys' do
+      raw = { foo: :bar, forbidden: 'ignored' }
+      expect { Rackstash::Fields::Hash(raw, forbidden_fields: ['forbidden']) }.to raise_error ArgumentError
+    end
+  end
+end

spec/rackstash_spec.rb

@@ -8,6 +8,7 @@ require 'spec_helper'
 describe Rackstash do
   it 'defines PROGRAME with the correct version' do
     expect(Rackstash::PROGNAME).to match %r{\Arackstash/v\d+(\..+)*\z}
+    expect(Rackstash::PROGNAME).to be_frozen
   end
 
   it 'defines SEVERITIES constants' do
@@ -20,4 +21,14 @@ describe Rackstash do
     expect(Rackstash::FATAL).to eql 4
     expect(Rackstash::UNKNOWN).to eql 5
   end
+
+  it 'defines EMPTY_* constants' do
+    expect(Rackstash::EMPTY_STRING).to eql ''
+    expect(Rackstash::EMPTY_STRING).to be_frozen
+
+    expect(Rackstash::EMPTY_SET).to eql Set.new
+    expect(Rackstash::EMPTY_SET).to be_frozen
+
+    expect(Rackstash::ISO8601_PRECISION).to be_a Integer
+  end
 end