From 6d4106fcb7785f62dae85332cf0cffbf7edc9c85 Mon Sep 17 00:00:00 2001 From: Holger Just Date: Thu, 25 Jan 2018 10:52:42 +0100 Subject: [PATCH] Add Select and Remove filters to filter which fields get passed to the encoder --- lib/rackstash/filter/remove.rb | 56 ++++++++++++++++++++++ lib/rackstash/filter/select.rb | 71 ++++++++++++++++++++++++++++ spec/rackstash/filter/remove_spec.rb | 52 ++++++++++++++++++++ spec/rackstash/filter/select_spec.rb | 58 +++++++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 lib/rackstash/filter/remove.rb create mode 100644 lib/rackstash/filter/select.rb create mode 100644 spec/rackstash/filter/remove_spec.rb create mode 100644 spec/rackstash/filter/select_spec.rb diff --git a/lib/rackstash/filter/remove.rb b/lib/rackstash/filter/remove.rb new file mode 100644 index 0000000..3b9c884 --- /dev/null +++ b/lib/rackstash/filter/remove.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true +# +# Copyright 2018 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'rackstash/filter' +require 'rackstash/helpers/utf8' + +module Rackstash + module Filter + # Delete all key-value pair from the given event where the key matches any + # of the field matchers. All other key-value pairs will be retained. + # + # @example + # Rackstash::Flow.new(STDOUT) do + # filter :remove, 'api_key', 'runtime' + # # ^^^^^^^ You can also use :delete here + # end + class Remove + include Rackstash::Helpers::UTF8 + + # @param field_matchers [Array] the fields + # to remove from the event. You can specify this in a varienty of ways, + # usually as a `String` or `Symbol` (which is compared to the key) or + # as a `Regexp`, `Proc` (which gets passed the key to check for + # inclusion) or any other object responding to the `===` method. You can + # also provide a block which is used as an additional `Proc` matcher in + # this case. + def initialize(*field_matchers) + keys, matchers = field_matchers.flatten.partition { |field| + field.is_a?(String) || field.is_a?(Symbol) + } + + @keys = Set[*keys.map! { |key| utf8_encode(key) }] + @matchers = matchers + @matchers << block if block_given? + end + + # Remove all key-value pairs in the given event where the key matches any + # of the configured field names. + # + # @param event [Hash] an event hash + # @return [Hash] the given `event` with all matching fields removed + def call(event) + event.delete_if { |key, _value| + @keys.include?(key) || @matchers.any? { |matcher| matcher === key } + } + event + end + end + + register Remove, :remove, :delete + end +end diff --git a/lib/rackstash/filter/select.rb b/lib/rackstash/filter/select.rb new file mode 100644 index 0000000..3de56ee --- /dev/null +++ b/lib/rackstash/filter/select.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true +# +# Copyright 2018 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'set' + +require 'rackstash/filter' +require 'rackstash/helpers/utf8' + +module Rackstash + module Filter + # Only keep named fields on the top-level of the given event. All key-value + # pairs in the given event with a key matching any of field matchers given + # in {#initialize} will be kept. Conversely, all fields matching none of the + # field matchers will be removed from the event. + # + # @example + # Rackstash::Flow.new(STDOUT) do + # filter :select, + # 'message', + # '@timestamp', + # 'tags', + # /\Ahttp_/, # keep all fields beginning with "http_" + # ->(key) { key.length < 5 } # keep all fields with short keys + # end + # + # With the example above, we are keeping the default fields, as well as any + # keys starting with `"http_"` or have short names. Here, we are thus + # retaining e.g. fields named `"http_path"` or `"uuid"`. Fields like + # `"user_name"`, `"webserver"`, or `"robot_arm"` will be removed from the + # event however since they don't match any of the configured matchers. + class Select + include Rackstash::Helpers::UTF8 + + # @param field_matchers [Array] the fields + # to keep in the event. You can specify this in a varienty of ways, + # usually as a `String` or `Symbol` (which is compared to the key) or + # as a `Regexp`, `Proc` (which gets passed the key to check for + # inclusion) or any other object responding to the `===` method. You can + # also provide a block which is used as an additional `Proc` matcher in + # this case. + def initialize(*field_matchers, &block) + keys, matchers = field_matchers.flatten.partition { |field| + field.is_a?(String) || field.is_a?(Symbol) + } + + @keys = Set[*keys.map! { |key| utf8_encode(key) }] + @matchers = matchers + @matchers << block if block_given? + end + + # Keep only key-value pairs in the given event where the key matches any + # of the configured field names. + # + # @param event [Hash] an event hash + # @return [Hash] the given `event` with only matching fields retained and + # non-matching fields removed + def call(event) + event.keep_if { |key, _value| + @keys.include?(key) || @matchers.any? { |matcher| matcher === key } + } + event + end + end + + register Select, :select + end +end diff --git a/spec/rackstash/filter/remove_spec.rb b/spec/rackstash/filter/remove_spec.rb new file mode 100644 index 0000000..cd450f6 --- /dev/null +++ b/spec/rackstash/filter/remove_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true +# +# Copyright 2018 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'spec_helper' + +require 'rackstash/filter/remove' + +describe Rackstash::Filter::Remove do + let(:event) { + { + 'foo' => 'foo value', + 'bar' => 'bar value' + } + } + + def filter!(*spec, &block) + described_class.new(*spec, &block).call(event) + end + + it 'removes existing fields' do + filter!('foo') + expect(event).to eql 'bar' => 'bar value' + end + + it 'it ignores missing fields' do + filter!('unknown', 'bar') + expect(event).to eql 'foo' => 'foo value' + end + + it 'stringifies spec values' do + filter!(:foo) + expect(event).to eql 'bar' => 'bar value' + end + + it 'removes fields matched by a regular expression' do + filter!(/f/, /blar/) + expect(event).to eql 'bar' => 'bar value' + end + + it 'removes fields matched by a Proc' do + filter!(->(key) { key.start_with?('b') }) + expect(event).to eql 'foo' => 'foo value' + end + + it 'returns the given event object' do + expect(filter!('bar')).to equal event + end +end diff --git a/spec/rackstash/filter/select_spec.rb b/spec/rackstash/filter/select_spec.rb new file mode 100644 index 0000000..d172c0b --- /dev/null +++ b/spec/rackstash/filter/select_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true +# +# Copyright 2018 Holger Just +# +# This software may be modified and distributed under the terms +# of the MIT license. See the LICENSE.txt file for details. + +require 'spec_helper' + +require 'rackstash/filter/select' + +describe Rackstash::Filter::Select do + let(:event) { + { + 'foo' => 'foo value', + 'bar' => 'bar value' + } + } + + def filter!(*spec, &block) + described_class.new(*spec, &block).call(event) + end + + it 'retains only matching fields' do + filter!('foo') + expect(event).to eql 'foo' => 'foo value' + end + + it 'it ignores missing fields' do + filter!('foo', 'unknown') + expect(event).to eql 'foo' => 'foo value' + end + + it 'stringifies spec values' do + filter!(:foo) + expect(event).to eql 'foo' => 'foo value' + end + + it 'retains fields matched by a regular expression' do + filter!(/b/, /blar/) + expect(event).to eql 'bar' => 'bar value' + end + + it 'retains fields matched by a Proc' do + filter!(->(key) { key.start_with?('b') }) + expect(event).to eql 'bar' => 'bar value' + end + + it 'retaines fields matched by the block' do + filter! { |key| key.start_with?('b') } + expect(event).to eql 'bar' => 'bar value' + end + + + it 'returns the given event object' do + expect(filter!('bar')).to equal event + end +end