diff --git a/lib/rackstash.rb b/lib/rackstash.rb index 051ed06..41c4349 100644 --- a/lib/rackstash.rb +++ b/lib/rackstash.rb @@ -86,6 +86,35 @@ module Rackstash SEVERITY_LABELS[severity] end end + + # Returns a {Flow} which is used by the normal logger {Flow}s to write details + # about any unexpected errors during interaction with their {Adapters}. + # + # By default, this Flow logs JSON-formatted messages to `STDERR` + # + # @return [Rackstash::Flow] the default error flow + def self.error_flow + @error_flow ||= Rackstash::Flow.new(STDERR) + end + + # Set a {Flow} which is used bythe normal logger {Flow}s to write details + # of any unexpected errors during interaction with their {Adapters}. + # + # You can set a different `error_flow` for each {Flow} if required. You can + # also change this flow to match your desired fallback format and log adapter. + # + # To still work in the face of unexpected availability issues like a full + # filesystem, an unavailable network, broken external loggers, or any other + # external issues, it is usually desireable to chose a local and mostly + # relibable log target. + # + # @param flow [Flow, Adapters::Adapter, Object] a single {Flow} or an object + # which can be used as a {Flow}'s adapter. See {Flow#initialize}. + # @return [Rackstash::Flow] the given `flow` + def self.error_flow=(flow) + flow = Flow.new(flow) unless flow.is_a?(Rackstash::Flow) + @error_flow = flow + end end require 'rackstash/logger' diff --git a/lib/rackstash/flow.rb b/lib/rackstash/flow.rb index f344dc3..8ca334a 100644 --- a/lib/rackstash/flow.rb +++ b/lib/rackstash/flow.rb @@ -76,11 +76,17 @@ module Rackstash # @yieldparam flow [self] if the given block accepts an argument, we yield # `self` as a parameter, else, the block is directly executed in the # context of `self`. - def initialize(adapter, encoder: nil, filters: [], &block) + def initialize(adapter, encoder: nil, filters: [], error_flow: nil, &block) @adapter = Rackstash::Adapters[adapter] self.encoder(encoder || @adapter.default_encoder) @filter_chain = Rackstash::FilterChain.new(filters) + if error_flow.nil? + @error_flow = nil + else + self.error_flow(error_flow) + end + if block_given? if block.arity == 0 instance_eval(&block) @@ -126,6 +132,15 @@ module Rackstash @encoder = encoder end + def error_flow(flow = nil) + if flow.nil? + @error_flow || Rackstash.error_flow + else + flow = Flow.new(flow) unless flow.is_a?(Rackstash::Flow) + @error_flow = flow + end + end + # (see FilterChain#insert_after) def filter_after(index, filter = nil, &block) @filter_chain.insert_after(index, filter, &block) @@ -236,9 +251,25 @@ module Rackstash private - # TODO: use a fallback flow and send formatted logs there def log_error(message, exception) - warn("#{message}: #{exception}") + error_event = { + FIELD_ERROR => exception.class.name, + FIELD_ERROR_MESSAGE => exception.message, + FIELD_ERROR_TRACE => (exception.backtrace || []).join("\n"), + + FIELD_TAGS => [], + FIELD_MESSAGE => message, + FIELD_TIMESTAMP => Time.now.utc.iso8601(ISO8601_PRECISION).freeze, + FIELD_VERSION => '1'.freeze + } + error_flow.write!(error_event) + rescue + # At this place, writing to the error log has also failed. This is a bad + # place to be in and there is very little we can sensibly do now. + # + # To aid in availability of the app using Rackstash, we swallow any + # StandardErrors here and just continue, hoping that things will turn out + # to be okay in the end. end end end diff --git a/spec/rackstash/flow_spec.rb b/spec/rackstash/flow_spec.rb index bdf74f1..648cfd7 100644 --- a/spec/rackstash/flow_spec.rb +++ b/spec/rackstash/flow_spec.rb @@ -75,9 +75,26 @@ describe Rackstash::Flow do end it 'rescues any exception thrown by the adapter' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!) + .with hash_including( + 'message' => /^close failed for adapter/, + 'error' => 'RuntimeError', + 'error_message' => 'ERROR' + ) + expect(flow).to receive(:error_flow).and_return(error_flow) + expect(flow).to receive(:close!).and_raise('ERROR') - expect(flow).to receive(:warn).with(/^close failed for adapter/) - flow.close + expect(flow.close).to be nil + end + + it 'rescues errors thrown by the error_flow' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!).and_raise('DOUBLE ERROR') + expect(flow).to receive(:error_flow).and_return(error_flow) + + expect(flow).to receive(:close!).and_raise('ERROR') + expect(flow.close).to be nil end end @@ -192,9 +209,26 @@ describe Rackstash::Flow do end it 'rescues any exception thrown by the adapter' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!) + .with hash_including( + 'message' => /^reopen failed for adapter/, + 'error' => 'RuntimeError', + 'error_message' => 'ERROR' + ) + expect(flow).to receive(:error_flow).and_return(error_flow) + expect(flow).to receive(:reopen!).and_raise('ERROR') - expect(flow).to receive(:warn).with(/^reopen failed for adapter/) - flow.reopen + expect(flow.reopen).to be nil + end + + it 'rescues errors thrown by the error_flow' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!).and_raise('DOUBLE ERROR') + expect(flow).to receive(:error_flow).and_return(error_flow) + + expect(flow).to receive(:reopen!).and_raise('ERROR') + expect(flow.reopen).to be nil end end @@ -263,9 +297,26 @@ describe Rackstash::Flow do end it 'rescues any exception thrown by the adapter' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!) + .with hash_including( + 'message' => /^write failed for adapter/, + 'error' => 'RuntimeError', + 'error_message' => 'ERROR' + ) + expect(flow).to receive(:error_flow).and_return(error_flow) + expect(flow).to receive(:write!).and_raise('ERROR') - expect(flow).to receive(:warn).with(/^write failed for adapter/) - flow.write(event) + expect(flow.write(event)).to be false + end + + it 'rescues errors thrown by the error_flow' do + error_flow = instance_double(described_class) + expect(error_flow).to receive(:write!).and_raise('DOUBLE ERROR') + expect(flow).to receive(:error_flow).and_return(error_flow) + + expect(flow).to receive(:write!).and_raise('ERROR') + expect(flow.write(event)).to be false end end end diff --git a/spec/rackstash_spec.rb b/spec/rackstash_spec.rb index 3a716a6..a509b7c 100644 --- a/spec/rackstash_spec.rb +++ b/spec/rackstash_spec.rb @@ -83,4 +83,38 @@ describe Rackstash do expect(described_class.severity_label(nil)).to eql 'ANY' end end + + describe '.error_flow' do + it 'returns a default Flow' do + expect(described_class.error_flow).to be_instance_of Rackstash::Flow + + expect(described_class.error_flow.encoder).to be_instance_of Rackstash::Encoders::JSON + expect(described_class.error_flow.adapter).to be_instance_of Rackstash::Adapters::IO + end + + it 'caches the flow' do + expect(described_class.error_flow).to equal described_class.error_flow + end + end + + describe '.error_flow=' do + let(:flow) { + flow = instance_double('Rackstash::Flow') + allow(flow).to receive(:is_a?).with(Rackstash::Flow).and_return(true) + flow + } + + it 'can set a new flow' do + described_class.error_flow = flow + expect(described_class.error_flow).to equal flow + end + + it 'wraps a non-flow' do + adapter = 'spec.log' + expect(Rackstash::Flow).to receive(:new).with(adapter).and_return(flow) + + described_class.error_flow = adapter + expect(described_class.error_flow).to equal flow + end + end end