Add Select and Remove filters to filter which fields get passed to the encoder

lib/rackstash/filter/remove.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+#
+# Copyright 2018 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'rackstash/filter'
+require 'rackstash/helpers/utf8'
+
+module Rackstash
+  module Filter
+    # Delete all key-value pair from the given event where the key matches any
+    # of the field matchers. All other key-value pairs will be retained.
+    #
+    # @example
+    #   Rackstash::Flow.new(STDOUT) do
+    #     filter :remove, 'api_key', 'runtime'
+    #     #      ^^^^^^^ You can also use :delete here
+    #   end
+    class Remove
+      include Rackstash::Helpers::UTF8
+
+      # @param field_matchers [Array<String,Symbol,Regexp,Proc,#===>] the fields
+      #   to remove from the event. You can specify this in a varienty of ways,
+      #   usually as a `String` or `Symbol` (which is compared to the key) or
+      #   as a `Regexp`, `Proc` (which gets passed the key to check for
+      #   inclusion) or any other object responding to the `===` method. You can
+      #   also provide a block which is used as an additional `Proc` matcher in
+      #   this case.
+      def initialize(*field_matchers)
+        keys, matchers = field_matchers.flatten.partition { |field|
+          field.is_a?(String) || field.is_a?(Symbol)
+        }
+
+        @keys = Set[*keys.map! { |key| utf8_encode(key) }]
+        @matchers = matchers
+        @matchers << block if block_given?
+      end
+
+      # Remove all key-value pairs in the given event where the key matches any
+      # of the configured field names.
+      #
+      # @param event [Hash] an event hash
+      # @return [Hash] the given `event` with all matching fields removed
+      def call(event)
+        event.delete_if { |key, _value|
+          @keys.include?(key) || @matchers.any? { |matcher| matcher === key }
+        }
+        event
+      end
+    end
+
+    register Remove, :remove, :delete
+  end
+end

lib/rackstash/filter/select.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+#
+# Copyright 2018 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'set'
+
+require 'rackstash/filter'
+require 'rackstash/helpers/utf8'
+
+module Rackstash
+  module Filter
+    # Only keep named fields on the top-level of the given event. All key-value
+    # pairs in the given event with a key matching any of field matchers given
+    # in {#initialize} will be kept. Conversely, all fields matching none of the
+    # field matchers will be removed from the event.
+    #
+    # @example
+    #   Rackstash::Flow.new(STDOUT) do
+    #     filter :select,
+    #       'message',
+    #       '@timestamp',
+    #       'tags',
+    #       /\Ahttp_/,                 # keep all fields beginning with "http_"
+    #       ->(key) { key.length < 5 } # keep all fields with short keys
+    #   end
+    #
+    # With the example above, we are keeping the default fields, as well as any
+    # keys starting with `"http_"` or have short names. Here, we are thus
+    # retaining e.g. fields named `"http_path"` or `"uuid"`. Fields like
+    # `"user_name"`, `"webserver"`, or `"robot_arm"` will be removed from the
+    # event however since they don't match any of the configured matchers.
+    class Select
+      include Rackstash::Helpers::UTF8
+
+      # @param field_matchers [Array<String,Symbol,Regexp,Proc,#===>] the fields
+      #   to keep in the event. You can specify this in a varienty of ways,
+      #   usually as a `String` or `Symbol` (which is compared to the key) or
+      #   as a `Regexp`, `Proc` (which gets passed the key to check for
+      #   inclusion) or any other object responding to the `===` method. You can
+      #   also provide a block which is used as an additional `Proc` matcher in
+      #   this case.
+      def initialize(*field_matchers, &block)
+        keys, matchers = field_matchers.flatten.partition { |field|
+          field.is_a?(String) || field.is_a?(Symbol)
+        }
+
+        @keys = Set[*keys.map! { |key| utf8_encode(key) }]
+        @matchers = matchers
+        @matchers << block if block_given?
+      end
+
+      # Keep only key-value pairs in the given event where the key matches any
+      # of the configured field names.
+      #
+      # @param event [Hash] an event hash
+      # @return [Hash] the given `event` with only matching fields retained and
+      #   non-matching fields removed
+      def call(event)
+        event.keep_if { |key, _value|
+          @keys.include?(key) || @matchers.any? { |matcher| matcher === key }
+        }
+        event
+      end
+    end
+
+    register Select, :select
+  end
+end

spec/rackstash/filter/remove_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+#
+# Copyright 2018 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'spec_helper'
+
+require 'rackstash/filter/remove'
+
+describe Rackstash::Filter::Remove do
+  let(:event) {
+    {
+      'foo' => 'foo value',
+      'bar' => 'bar value'
+    }
+  }
+
+  def filter!(*spec, &block)
+    described_class.new(*spec, &block).call(event)
+  end
+
+  it 'removes existing fields' do
+    filter!('foo')
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+  it 'it ignores missing fields' do
+    filter!('unknown', 'bar')
+    expect(event).to eql 'foo' => 'foo value'
+  end
+
+  it 'stringifies spec values' do
+    filter!(:foo)
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+  it 'removes fields matched by a regular expression' do
+    filter!(/f/, /blar/)
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+  it 'removes fields matched by a Proc' do
+    filter!(->(key) { key.start_with?('b') })
+    expect(event).to eql 'foo' => 'foo value'
+  end
+
+  it 'returns the given event object' do
+    expect(filter!('bar')).to equal event
+  end
+end

spec/rackstash/filter/select_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+#
+# Copyright 2018 Holger Just
+#
+# This software may be modified and distributed under the terms
+# of the MIT license. See the LICENSE.txt file for details.
+
+require 'spec_helper'
+
+require 'rackstash/filter/select'
+
+describe Rackstash::Filter::Select do
+  let(:event) {
+    {
+      'foo' => 'foo value',
+      'bar' => 'bar value'
+    }
+  }
+
+  def filter!(*spec, &block)
+    described_class.new(*spec, &block).call(event)
+  end
+
+  it 'retains only matching fields' do
+    filter!('foo')
+    expect(event).to eql 'foo' => 'foo value'
+  end
+
+  it 'it ignores missing fields' do
+    filter!('foo', 'unknown')
+    expect(event).to eql 'foo' => 'foo value'
+  end
+
+  it 'stringifies spec values' do
+    filter!(:foo)
+    expect(event).to eql 'foo' => 'foo value'
+  end
+
+  it 'retains fields matched by a regular expression' do
+    filter!(/b/, /blar/)
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+  it 'retains fields matched by a Proc' do
+    filter!(->(key) { key.start_with?('b') })
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+  it 'retaines fields matched by the block' do
+    filter! { |key| key.start_with?('b') }
+    expect(event).to eql 'bar' => 'bar value'
+  end
+
+
+  it 'returns the given event object' do
+    expect(filter!('bar')).to equal event
+  end
+end