1 // Copyright Joyent, Inc. and other Node contributors.
3 // Permission is hereby granted, free of charge, to any person obtaining a
4 // copy of this software and associated documentation files (the
5 // "Software"), to deal in the Software without restriction, including
6 // without limitation the rights to use, copy, modify, merge, publish,
7 // distribute, sublicense, and/or sell copies of the Software, and to permit
8 // persons to whom the Software is furnished to do so, subject to the
9 // following conditions:
11 // The above copyright notice and this permission notice shall be included
12 // in all copies or substantial portions of the Software.
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17 // NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18 // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 // USE OR OTHER DEALINGS IN THE SOFTWARE.
22 module.exports = Readable;
25 var isArray = require('isarray');
30 var Buffer = require('buffer').Buffer;
33 Readable.ReadableState = ReadableState;
35 var EE = require('events').EventEmitter;
38 if (!EE.listenerCount) EE.listenerCount = function(emitter, type) {
39 return emitter.listeners(type).length;
43 var Stream = require('stream');
46 var util = require('core-util-is');
47 util.inherits = require('inherits');
52 util.inherits(Readable, Stream);
54 function ReadableState(options, stream) {
55 options = options || {};
57 // the point at which it stops calling _read() to fill the buffer
58 // Note: 0 is a valid value, means "don't call _read preemptively ever"
59 var hwm = options.highWaterMark;
60 this.highWaterMark = (hwm || hwm === 0) ? hwm : 16 * 1024;
63 this.highWaterMark = ~~this.highWaterMark;
71 this.endEmitted = false;
74 // In streams that never have any data, and do push(null) right away,
75 // the consumer can miss the 'end' event if they do some I/O before
76 // consuming the stream. So, we don't emit('end') until some reading
78 this.calledRead = false;
80 // a flag to be able to tell if the onwrite cb is called immediately,
81 // or on a later tick. We set this to true at first, becuase any
82 // actions that shouldn't happen until "later" should generally also
83 // not happen before the first write call.
86 // whenever we return null, then we set a flag to say
87 // that we're awaiting a 'readable' event emission.
88 this.needReadable = false;
89 this.emittedReadable = false;
90 this.readableListening = false;
93 // object stream flag. Used to make read(n) ignore n and to
94 // make all the buffer merging and length checks go away
95 this.objectMode = !!options.objectMode;
97 // Crypto is kind of old and crusty. Historically, its default string
98 // encoding is 'binary' so we have to make this configurable.
99 // Everything else in the universe uses 'utf8', though.
100 this.defaultEncoding = options.defaultEncoding || 'utf8';
102 // when piping, we only care about 'readable' events that happen
103 // after read()ing all the bytes and not getting any pushback.
106 // the number of writers that are awaiting a drain event in .pipe()s
109 // if true, a maybeReadMore has been scheduled
110 this.readingMore = false;
113 this.encoding = null;
114 if (options.encoding) {
116 StringDecoder = require('string_decoder/').StringDecoder;
117 this.decoder = new StringDecoder(options.encoding);
118 this.encoding = options.encoding;
122 function Readable(options) {
123 if (!(this instanceof Readable))
124 return new Readable(options);
126 this._readableState = new ReadableState(options, this);
129 this.readable = true;
134 // Manually shove something into the read() buffer.
135 // This returns true if the highWaterMark has not been hit yet,
136 // similar to how Writable.write() returns true if you should
137 // write() some more.
138 Readable.prototype.push = function(chunk, encoding) {
139 var state = this._readableState;
141 if (typeof chunk === 'string' && !state.objectMode) {
142 encoding = encoding || state.defaultEncoding;
143 if (encoding !== state.encoding) {
144 chunk = new Buffer(chunk, encoding);
149 return readableAddChunk(this, state, chunk, encoding, false);
152 // Unshift should *always* be something directly out of read()
153 Readable.prototype.unshift = function(chunk) {
154 var state = this._readableState;
155 return readableAddChunk(this, state, chunk, '', true);
158 function readableAddChunk(stream, state, chunk, encoding, addToFront) {
159 var er = chunkInvalid(state, chunk);
161 stream.emit('error', er);
162 } else if (chunk === null || chunk === undefined) {
163 state.reading = false;
165 onEofChunk(stream, state);
166 } else if (state.objectMode || chunk && chunk.length > 0) {
167 if (state.ended && !addToFront) {
168 var e = new Error('stream.push() after EOF');
169 stream.emit('error', e);
170 } else if (state.endEmitted && addToFront) {
171 var e = new Error('stream.unshift() after end event');
172 stream.emit('error', e);
174 if (state.decoder && !addToFront && !encoding)
175 chunk = state.decoder.write(chunk);
177 // update the buffer info.
178 state.length += state.objectMode ? 1 : chunk.length;
180 state.buffer.unshift(chunk);
182 state.reading = false;
183 state.buffer.push(chunk);
186 if (state.needReadable)
187 emitReadable(stream);
189 maybeReadMore(stream, state);
191 } else if (!addToFront) {
192 state.reading = false;
195 return needMoreData(state);
200 // if it's past the high water mark, we can push in some more.
201 // Also, if we have no data yet, we can stand some
202 // more bytes. This is to work around cases where hwm=0,
203 // such as the repl. Also, if the push() triggered a
204 // readable event, and the user called read(largeNumber) such that
205 // needReadable was set, then we ought to push more, so that another
206 // 'readable' event will be triggered.
207 function needMoreData(state) {
208 return !state.ended &&
209 (state.needReadable ||
210 state.length < state.highWaterMark ||
214 // backwards compatibility.
215 Readable.prototype.setEncoding = function(enc) {
217 StringDecoder = require('string_decoder/').StringDecoder;
218 this._readableState.decoder = new StringDecoder(enc);
219 this._readableState.encoding = enc;
222 // Don't raise the hwm > 128MB
223 var MAX_HWM = 0x800000;
224 function roundUpToNextPowerOf2(n) {
228 // Get the next highest power of 2
230 for (var p = 1; p < 32; p <<= 1) n |= n >> p;
236 function howMuchToRead(n, state) {
237 if (state.length === 0 && state.ended)
240 if (state.objectMode)
241 return n === 0 ? 0 : 1;
243 if (n === null || isNaN(n)) {
244 // only flow one buffer at a time
245 if (state.flowing && state.buffer.length)
246 return state.buffer[0].length;
254 // If we're asking for more than the target buffer level,
255 // then raise the water mark. Bump up to the next highest
256 // power of 2, to prevent increasing it excessively in tiny
258 if (n > state.highWaterMark)
259 state.highWaterMark = roundUpToNextPowerOf2(n);
261 // don't have that much. return null, unless we've ended.
262 if (n > state.length) {
264 state.needReadable = true;
273 // you can override either this method, or the async _read(n) below.
274 Readable.prototype.read = function(n) {
275 var state = this._readableState;
276 state.calledRead = true;
280 if (typeof n !== 'number' || n > 0)
281 state.emittedReadable = false;
283 // if we're doing read(0) to trigger a readable event, but we
284 // already have a bunch of data in the buffer, then just trigger
285 // the 'readable' event and move on.
287 state.needReadable &&
288 (state.length >= state.highWaterMark || state.ended)) {
293 n = howMuchToRead(n, state);
295 // if we've ended, and we're now clear, then finish it up.
296 if (n === 0 && state.ended) {
299 // In cases where the decoder did not receive enough data
300 // to produce a full chunk, then immediately received an
301 // EOF, state.buffer will contain [<Buffer >, <Buffer 00 ...>].
302 // howMuchToRead will see this and coerce the amount to
303 // read to zero (because it's looking at the length of the
304 // first <Buffer > in state.buffer), and we'll end up here.
306 // This can only happen via state.decoder -- no other venue
307 // exists for pushing a zero-length chunk into state.buffer
308 // and triggering this behavior. In this case, we return our
309 // remaining data and end the stream, if appropriate.
310 if (state.length > 0 && state.decoder) {
311 ret = fromList(n, state);
312 state.length -= ret.length;
315 if (state.length === 0)
321 // All the actual chunk generation logic needs to be
322 // *below* the call to _read. The reason is that in certain
323 // synthetic stream cases, such as passthrough streams, _read
324 // may be a completely synchronous operation which may change
325 // the state of the read buffer, providing enough data when
326 // before there was *not* enough.
328 // So, the steps are:
329 // 1. Figure out what the state of things will be after we do
330 // a read from the buffer.
332 // 2. If that resulting state will trigger a _read, then call _read.
333 // Note that this may be asynchronous, or synchronous. Yes, it is
334 // deeply ugly to write APIs this way, but that still doesn't mean
335 // that the Readable class should behave improperly, as streams are
336 // designed to be sync/async agnostic.
337 // Take note if the _read call is sync or async (ie, if the read call
338 // has returned yet), so that we know whether or not it's safe to emit
341 // 3. Actually pull the requested chunks out of the buffer and return.
343 // if we need a readable event, then we need to do some reading.
344 var doRead = state.needReadable;
346 // if we currently have less than the highWaterMark, then also read some
347 if (state.length - n <= state.highWaterMark)
350 // however, if we've ended, then there's no point, and if we're already
351 // reading, then it's unnecessary.
352 if (state.ended || state.reading)
356 state.reading = true;
358 // if the length is currently zero, then we *need* a readable event.
359 if (state.length === 0)
360 state.needReadable = true;
361 // call internal read method
362 this._read(state.highWaterMark);
366 // If _read called its callback synchronously, then `reading`
367 // will be false, and we need to re-evaluate how much data we
368 // can return to the user.
369 if (doRead && !state.reading)
370 n = howMuchToRead(nOrig, state);
373 ret = fromList(n, state);
378 state.needReadable = true;
384 // If we have nothing in the buffer, then we want to know
385 // as soon as we *do* get something into the buffer.
386 if (state.length === 0 && !state.ended)
387 state.needReadable = true;
389 // If we happened to read() exactly the remaining amount in the
390 // buffer, and the EOF has been seen at this point, then make sure
391 // that we emit 'end' on the very next tick.
392 if (state.ended && !state.endEmitted && state.length === 0)
398 function chunkInvalid(state, chunk) {
400 if (!Buffer.isBuffer(chunk) &&
401 'string' !== typeof chunk &&
403 chunk !== undefined &&
405 er = new TypeError('Invalid non-string/buffer chunk');
411 function onEofChunk(stream, state) {
412 if (state.decoder && !state.ended) {
413 var chunk = state.decoder.end();
414 if (chunk && chunk.length) {
415 state.buffer.push(chunk);
416 state.length += state.objectMode ? 1 : chunk.length;
421 // if we've ended and we have some data left, then emit
422 // 'readable' now to make sure it gets picked up.
423 if (state.length > 0)
424 emitReadable(stream);
429 // Don't emit readable right away in sync mode, because this can trigger
430 // another read() call => stack overflow. This way, it might trigger
431 // a nextTick recursion warning, but that's not so bad.
432 function emitReadable(stream) {
433 var state = stream._readableState;
434 state.needReadable = false;
435 if (state.emittedReadable)
438 state.emittedReadable = true;
440 process.nextTick(function() {
441 emitReadable_(stream);
444 emitReadable_(stream);
447 function emitReadable_(stream) {
448 stream.emit('readable');
452 // at this point, the user has presumably seen the 'readable' event,
453 // and called read() to consume some data. that may have triggered
454 // in turn another _read(n) call, in which case reading = true if
456 // However, if we're not ended, or reading, and the length < hwm,
457 // then go ahead and try to read some more preemptively.
458 function maybeReadMore(stream, state) {
459 if (!state.readingMore) {
460 state.readingMore = true;
461 process.nextTick(function() {
462 maybeReadMore_(stream, state);
467 function maybeReadMore_(stream, state) {
468 var len = state.length;
469 while (!state.reading && !state.flowing && !state.ended &&
470 state.length < state.highWaterMark) {
472 if (len === state.length)
473 // didn't get any data, stop spinning.
478 state.readingMore = false;
481 // abstract method. to be overridden in specific implementation classes.
482 // call cb(er, data) where data is <= n in length.
483 // for virtual (non-string, non-buffer) streams, "length" is somewhat
484 // arbitrary, and perhaps not very meaningful.
485 Readable.prototype._read = function(n) {
486 this.emit('error', new Error('not implemented'));
489 Readable.prototype.pipe = function(dest, pipeOpts) {
491 var state = this._readableState;
493 switch (state.pipesCount) {
498 state.pipes = [state.pipes, dest];
501 state.pipes.push(dest);
504 state.pipesCount += 1;
506 var doEnd = (!pipeOpts || pipeOpts.end !== false) &&
507 dest !== process.stdout &&
508 dest !== process.stderr;
510 var endFn = doEnd ? onend : cleanup;
511 if (state.endEmitted)
512 process.nextTick(endFn);
514 src.once('end', endFn);
516 dest.on('unpipe', onunpipe);
517 function onunpipe(readable) {
518 if (readable !== src) return;
526 // when the dest drains, it reduces the awaitDrain counter
527 // on the source. This would be more elegant with a .once()
528 // handler in flow(), but adding and removing repeatedly is
530 var ondrain = pipeOnDrain(src);
531 dest.on('drain', ondrain);
534 // cleanup event handlers once the pipe is broken
535 dest.removeListener('close', onclose);
536 dest.removeListener('finish', onfinish);
537 dest.removeListener('drain', ondrain);
538 dest.removeListener('error', onerror);
539 dest.removeListener('unpipe', onunpipe);
540 src.removeListener('end', onend);
541 src.removeListener('end', cleanup);
543 // if the reader is waiting for a drain event from this
544 // specific writer, then it would cause it to never start
546 // So, if this is awaiting a drain, then we just call it now.
547 // If we don't know, then assume that we are waiting for one.
548 if (!dest._writableState || dest._writableState.needDrain)
552 // if the dest has an error, then stop piping into it.
553 // however, don't suppress the throwing behavior for this.
554 function onerror(er) {
556 dest.removeListener('error', onerror);
557 if (EE.listenerCount(dest, 'error') === 0)
558 dest.emit('error', er);
560 // This is a brutally ugly hack to make sure that our error handler
561 // is attached before any userland ones. NEVER DO THIS.
562 if (!dest._events || !dest._events.error)
563 dest.on('error', onerror);
564 else if (isArray(dest._events.error))
565 dest._events.error.unshift(onerror);
567 dest._events.error = [onerror, dest._events.error];
571 // Both close and finish should trigger unpipe, but only once.
573 dest.removeListener('finish', onfinish);
576 dest.once('close', onclose);
577 function onfinish() {
578 dest.removeListener('close', onclose);
581 dest.once('finish', onfinish);
587 // tell the dest that it's being piped to
588 dest.emit('pipe', src);
590 // start the flow if it hasn't been started already.
591 if (!state.flowing) {
592 // the handler that waits for readable events after all
593 // the data gets sucked out in flow.
594 // This would be easier to follow with a .once() handler
595 // in flow(), but that is too slow.
596 this.on('readable', pipeOnReadable);
598 state.flowing = true;
599 process.nextTick(function() {
607 function pipeOnDrain(src) {
610 var state = src._readableState;
612 if (state.awaitDrain === 0)
618 var state = src._readableState;
620 state.awaitDrain = 0;
622 function write(dest, i, list) {
623 var written = dest.write(chunk);
624 if (false === written) {
629 while (state.pipesCount && null !== (chunk = src.read())) {
631 if (state.pipesCount === 1)
632 write(state.pipes, 0, null);
634 forEach(state.pipes, write);
636 src.emit('data', chunk);
638 // if anyone needs a drain, then we have to wait for that.
639 if (state.awaitDrain > 0)
643 // if every destination was unpiped, either before entering this
644 // function, or in the while loop, then stop flowing.
646 // NB: This is a pretty rare edge case.
647 if (state.pipesCount === 0) {
648 state.flowing = false;
650 // if there were data event listeners added, then switch to old mode.
651 if (EE.listenerCount(src, 'data') > 0)
656 // at this point, no one needed a drain, so we just ran out of data
657 // on the next readable event, start it over again.
661 function pipeOnReadable() {
662 if (this._readableState.ranOut) {
663 this._readableState.ranOut = false;
669 Readable.prototype.unpipe = function(dest) {
670 var state = this._readableState;
672 // if we're not piping anywhere, then do nothing.
673 if (state.pipesCount === 0)
676 // just one destination. most common case.
677 if (state.pipesCount === 1) {
678 // passed in one, but it's not the right one.
679 if (dest && dest !== state.pipes)
687 state.pipesCount = 0;
688 this.removeListener('readable', pipeOnReadable);
689 state.flowing = false;
691 dest.emit('unpipe', this);
695 // slow case. multiple pipe destinations.
699 var dests = state.pipes;
700 var len = state.pipesCount;
702 state.pipesCount = 0;
703 this.removeListener('readable', pipeOnReadable);
704 state.flowing = false;
706 for (var i = 0; i < len; i++)
707 dests[i].emit('unpipe', this);
711 // try to find the right one.
712 var i = indexOf(state.pipes, dest);
716 state.pipes.splice(i, 1);
717 state.pipesCount -= 1;
718 if (state.pipesCount === 1)
719 state.pipes = state.pipes[0];
721 dest.emit('unpipe', this);
726 // set up data events if they are asked for
727 // Ensure readable listeners eventually get something
728 Readable.prototype.on = function(ev, fn) {
729 var res = Stream.prototype.on.call(this, ev, fn);
731 if (ev === 'data' && !this._readableState.flowing)
732 emitDataEvents(this);
734 if (ev === 'readable' && this.readable) {
735 var state = this._readableState;
736 if (!state.readableListening) {
737 state.readableListening = true;
738 state.emittedReadable = false;
739 state.needReadable = true;
740 if (!state.reading) {
742 } else if (state.length) {
743 emitReadable(this, state);
750 Readable.prototype.addListener = Readable.prototype.on;
752 // pause() and resume() are remnants of the legacy readable stream API
753 // If the user uses them, then switch into old mode.
754 Readable.prototype.resume = function() {
755 emitDataEvents(this);
760 Readable.prototype.pause = function() {
761 emitDataEvents(this, true);
765 function emitDataEvents(stream, startPaused) {
766 var state = stream._readableState;
769 // https://github.com/isaacs/readable-stream/issues/16
770 throw new Error('Cannot switch to old mode now.');
773 var paused = startPaused || false;
774 var readable = false;
776 // convert to an old-style stream.
777 stream.readable = true;
778 stream.pipe = Stream.prototype.pipe;
779 stream.on = stream.addListener = Stream.prototype.on;
781 stream.on('readable', function() {
785 while (!paused && (null !== (c = stream.read())))
786 stream.emit('data', c);
790 stream._readableState.needReadable = true;
794 stream.pause = function() {
799 stream.resume = function() {
802 process.nextTick(function() {
803 stream.emit('readable');
810 // now make it start, just in case it hadn't already.
811 stream.emit('readable');
814 // wrap an old-style stream as the async data source.
815 // This is *not* part of the readable stream interface.
816 // It is an ugly unfortunate mess of history.
817 Readable.prototype.wrap = function(stream) {
818 var state = this._readableState;
822 stream.on('end', function() {
823 if (state.decoder && !state.ended) {
824 var chunk = state.decoder.end();
825 if (chunk && chunk.length)
832 stream.on('data', function(chunk) {
834 chunk = state.decoder.write(chunk);
836 // don't skip over falsy values in objectMode
837 //if (state.objectMode && util.isNullOrUndefined(chunk))
838 if (state.objectMode && (chunk === null || chunk === undefined))
840 else if (!state.objectMode && (!chunk || !chunk.length))
843 var ret = self.push(chunk);
850 // proxy all the other methods.
851 // important when wrapping filters and duplexes.
852 for (var i in stream) {
853 if (typeof stream[i] === 'function' &&
854 typeof this[i] === 'undefined') {
855 this[i] = function(method) { return function() {
856 return stream[method].apply(stream, arguments);
861 // proxy certain important events.
862 var events = ['error', 'close', 'destroy', 'pause', 'resume'];
863 forEach(events, function(ev) {
864 stream.on(ev, self.emit.bind(self, ev));
867 // when we try to consume some more bytes, simply unpause the
868 // underlying stream.
869 self._read = function(n) {
881 // exposed for testing purposes only.
882 Readable._fromList = fromList;
884 // Pluck off n bytes from an array of buffers.
885 // Length is the combined lengths of all the buffers in the list.
886 function fromList(n, state) {
887 var list = state.buffer;
888 var length = state.length;
889 var stringMode = !!state.decoder;
890 var objectMode = !!state.objectMode;
893 // nothing in the list, definitely empty.
894 if (list.length === 0)
901 else if (!n || n >= length) {
902 // read it all, truncate the array.
906 ret = Buffer.concat(list, length);
909 // read just some of it.
910 if (n < list[0].length) {
911 // just take a part of the first list item.
912 // slice is the same for buffers and strings.
914 ret = buf.slice(0, n);
915 list[0] = buf.slice(n);
916 } else if (n === list[0].length) {
917 // first list is a perfect match
921 // we have enough to cover it, but it spans past the first buffer.
928 for (var i = 0, l = list.length; i < l && c < n; i++) {
930 var cpy = Math.min(n - c, buf.length);
933 ret += buf.slice(0, cpy);
935 buf.copy(ret, c, 0, cpy);
937 if (cpy < buf.length)
938 list[0] = buf.slice(cpy);
950 function endReadable(stream) {
951 var state = stream._readableState;
953 // If we get here before consuming all the bytes, then that is a
954 // bug in node. Should never happen.
955 if (state.length > 0)
956 throw new Error('endReadable called on non-empty stream');
958 if (!state.endEmitted && state.calledRead) {
960 process.nextTick(function() {
961 // Check that we didn't get one last unshift.
962 if (!state.endEmitted && state.length === 0) {
963 state.endEmitted = true;
964 stream.readable = false;
971 function forEach (xs, f) {
972 for (var i = 0, l = xs.length; i < l; i++) {
977 function indexOf (xs, x) {
978 for (var i = 0, l = xs.length; i < l; i++) {
979 if (xs[i] === x) return i;