OpenShot Library | libopenshot 0.3.1
FrameMapper.cpp
Go to the documentation of this file.
1
9// Copyright (c) 2008-2019 OpenShot Studios, LLC
10//
11// SPDX-License-Identifier: LGPL-3.0-or-later
12
13#include <cmath>
14#include <iostream>
15#include <iomanip>
16
17#include "FrameMapper.h"
18#include "Exceptions.h"
19#include "Clip.h"
20#include "ZmqLogger.h"
21
22using namespace std;
23using namespace openshot;
24
25FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
26 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0), previous_frame(0)
27{
28 // Set the original frame rate from the reader
29 original = Fraction(reader->info.fps.num, reader->info.fps.den);
30
31 // Set all info struct members equal to the internal reader
32 info = reader->info;
33 info.fps.num = target.num;
34 info.fps.den = target.den;
35 info.video_timebase.num = target.den;
36 info.video_timebase.den = target.num;
38 info.sample_rate = target_sample_rate;
39 info.channels = target_channels;
40 info.channel_layout = target_channel_layout;
41 info.width = reader->info.width;
42 info.height = reader->info.height;
43
44 // Enable/Disable audio (based on settings)
46
47 // Used to toggle odd / even fields
48 field_toggle = true;
49
50 // Adjust cache size based on size of frame and audio
52}
53
54// Destructor
56
57 // Auto Close if not already
58 Close();
59
60 reader = NULL;
61}
62
65{
66 if (reader)
67 return reader;
68 else
69 // Throw error if reader not initialized
70 throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
71}
72
73void FrameMapper::AddField(int64_t frame)
74{
75 // Add a field, and toggle the odd / even field
76 AddField(Field(frame, field_toggle));
77}
78
79void FrameMapper::AddField(Field field)
80{
81 // Add a field to the end of the field list
82 fields.push_back(field);
83
84 // toggle the odd / even flag
85 field_toggle = (field_toggle ? false : true);
86}
87
88// Clear both the fields & frames lists
89void FrameMapper::Clear() {
90 // Prevent async calls to the following code
91 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
92
93 // Clear the fields & frames lists
94 fields.clear();
95 fields.shrink_to_fit();
96 frames.clear();
97 frames.shrink_to_fit();
98}
99
100// Use the original and target frame rates and a pull-down technique to create
101// a mapping between the original fields and frames or a video to a new frame rate.
102// This might repeat or skip fields and frames of the original video, depending on
103// whether the frame rate is increasing or decreasing.
104void FrameMapper::Init()
105{
106 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
107
108 // Do not initialize anything if just a picture with no audio
110 // Skip initialization
111 return;
112
113 // Prevent async calls to the following code
114 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
115
116 // Clear the fields & frames lists
117 Clear();
118
119 // Find parent position (if any)
120 Clip *parent = static_cast<Clip *>(ParentClip());
121 if (parent) {
122 parent_position = parent->Position();
123 parent_start = parent->Start();
124 } else {
125 parent_position = 0.0;
126 parent_start = 0.0;
127 }
128
129 // Mark as not dirty
130 is_dirty = false;
131
132 // Clear cache
133 final_cache.Clear();
134
135 // Some framerates are handled special, and some use a generic Keyframe curve to
136 // map the framerates. These are the special framerates:
137 if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
138 (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
139
140 // Get the difference (in frames) between the original and target frame rates
141 float difference = target.ToInt() - original.ToInt();
142
143 // Find the number (i.e. interval) of fields that need to be skipped or repeated
144 int field_interval = 0;
145 int frame_interval = 0;
146
147 if (difference != 0)
148 {
149 field_interval = round(fabs(original.ToInt() / difference));
150
151 // Get frame interval (2 fields per frame)
152 frame_interval = field_interval * 2.0f;
153 }
154
155
156 // Calculate # of fields to map
157 int64_t frame = 1;
158 int64_t number_of_fields = reader->info.video_length * 2;
159
160 // Loop through all fields in the original video file
161 for (int64_t field = 1; field <= number_of_fields; field++)
162 {
163
164 if (difference == 0) // Same frame rate, NO pull-down or special techniques required
165 {
166 // Add fields
167 AddField(frame);
168 }
169 else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
170 {
171 // Add current field
172 AddField(frame);
173
174 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
175 {
176 // Add extra field for each 'field interval
177 AddField(frame);
178 }
179 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
180 {
181 // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
182 AddField(frame); // add field for current frame
183
184 if (frame + 1 <= info.video_length)
185 // add field for next frame (if the next frame exists)
186 AddField(Field(frame + 1, field_toggle));
187 }
188 else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
189 {
190 // No pull-down technique needed, just repeat this frame
191 AddField(frame);
192 AddField(frame);
193 }
194 }
195 else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
196 {
197
198 if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
199 {
200 // skip current field and toggle the odd/even flag
201 field_toggle = (field_toggle ? false : true);
202 }
203 else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
204 {
205 // skip this field, plus the next field
206 field++;
207 }
208 else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
209 {
210 // skip this field, plus the next one
211 field++;
212 }
213 else
214 {
215 // No skipping needed, so add the field
216 AddField(frame);
217 }
218 }
219
220 // increment frame number (if field is divisible by 2)
221 if (field % 2 == 0 && field > 0)
222 frame++;
223 }
224
225 } else {
226 // Map the remaining framerates using a linear algorithm
227 double rate_diff = target.ToDouble() / original.ToDouble();
228 int64_t new_length = reader->info.video_length * rate_diff;
229
230 // Calculate the value difference
231 double value_increment = reader->info.video_length / (double) (new_length);
232
233 // Loop through curve, and build list of frames
234 double original_frame_num = 1.0f;
235 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
236 {
237 // Add 2 fields per frame
238 AddField(round(original_frame_num));
239 AddField(round(original_frame_num));
240
241 // Increment original frame number
242 original_frame_num += value_increment;
243 }
244 }
245
246 // Loop through the target frames again (combining fields into frames)
247 Field Odd(0, true); // temp field used to track the ODD field
248 Field Even(0, true); // temp field used to track the EVEN field
249
250 // Variables used to remap audio samples
251 int64_t start_samples_frame = 1;
252 int start_samples_position = 0;
253
254 for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
255 {
256 // Get the current field
257 Field f = fields[field - 1];
258
259 // Is field divisible by 2?
260 if (field % 2 == 0 && field > 0)
261 {
262 // New frame number
263 int64_t frame_number = field / 2;
264
265 // Set the bottom frame
266 if (f.isOdd)
267 Odd = f;
268 else
269 Even = f;
270
271 // Determine the range of samples (from the original rate). Resampling happens in real-time when
272 // calling the GetFrame() method. So this method only needs to redistribute the original samples with
273 // the original sample rate.
274 int64_t end_samples_frame = start_samples_frame;
275 int end_samples_position = start_samples_position;
276 int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels);
277
278 while (remaining_samples > 0)
279 {
280 // Get original samples (with NO framerate adjustments)
281 // This is the original reader's frame numbers
282 int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
283
284 // Enough samples
285 if (original_samples >= remaining_samples)
286 {
287 // Take all that we need, and break loop
288 end_samples_position += remaining_samples - 1;
289 remaining_samples = 0;
290 } else
291 {
292 // Not enough samples (take them all, and keep looping)
293 end_samples_frame += 1; // next frame
294 end_samples_position = 0; // next frame, starting on 1st sample
295 remaining_samples -= original_samples; // reduce the remaining amount
296 }
297 }
298
299
300
301 // Create the sample mapping struct
302 SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, reader->info.sample_rate, reader->info.channels)};
303
304 // Reset the audio variables
305 start_samples_frame = end_samples_frame;
306 start_samples_position = end_samples_position + 1;
307 if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame), original, reader->info.sample_rate, reader->info.channels))
308 {
309 start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
310 start_samples_position = 0; // reset to 0, since we wrapped
311 }
312
313 // Create a frame and ADD it to the frames collection
314 MappedFrame frame = {Odd, Even, Samples};
315 frames.push_back(frame);
316 }
317 else
318 {
319 // Set the top field
320 if (f.isOdd)
321 Odd = f;
322 else
323 Even = f;
324 }
325 }
326
327 // Clear the internal fields list (no longer needed)
328 fields.clear();
329 fields.shrink_to_fit();
330
331 if (avr) {
332 // Delete resampler (if exists)
333 SWR_CLOSE(avr);
334 SWR_FREE(&avr);
335 avr = NULL;
336 }
337}
338
339MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
340{
341 // Check if mappings are dirty (and need to be recalculated)
342 if (is_dirty)
343 // Recalculate mappings
344 Init();
345
346 // Ignore mapping on single image readers
348 // Return the same number
349 MappedFrame frame;
350 frame.Even.Frame = TargetFrameNumber;
351 frame.Odd.Frame = TargetFrameNumber;
352 frame.Samples.frame_start = 0;
353 frame.Samples.frame_end = 0;
354 frame.Samples.sample_start = 0;
355 frame.Samples.sample_end = 0;
356 frame.Samples.total = 0;
357 return frame;
358 }
359
360 // Check if frame number is valid
361 if(TargetFrameNumber < 1 || frames.size() == 0)
362 // frame too small, return error
363 throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
364
365 else if (TargetFrameNumber > (int64_t)frames.size())
366 // frame too large, set to end frame
367 TargetFrameNumber = frames.size();
368
369 // Debug output
371 "FrameMapper::GetMappedFrame",
372 "TargetFrameNumber", TargetFrameNumber,
373 "frames.size()", frames.size(),
374 "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame,
375 "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
376
377 // Return frame
378 return frames[TargetFrameNumber - 1];
379}
380
381// Get or generate a blank frame
382std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
383{
384 std::shared_ptr<Frame> new_frame;
385
386 // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
387 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number), target, reader->info.sample_rate, reader->info.channels);
388
389 try {
390 // Debug output
392 "FrameMapper::GetOrCreateFrame (from reader)",
393 "number", number,
394 "samples_in_frame", samples_in_frame);
395
396 // Attempt to get a frame (but this could fail if a reader has just been closed)
397 new_frame = reader->GetFrame(number);
398
399 // Return real frame
400 return new_frame;
401
402 } catch (const ReaderClosed & e) {
403 // ...
404 } catch (const OutOfBoundsFrame & e) {
405 // ...
406 }
407
408 // Debug output
410 "FrameMapper::GetOrCreateFrame (create blank)",
411 "number", number,
412 "samples_in_frame", samples_in_frame);
413
414 // Create blank frame
415 new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
416 new_frame->SampleRate(reader->info.sample_rate);
417 new_frame->ChannelsLayout(info.channel_layout);
418 new_frame->AddAudioSilence(samples_in_frame);
419 return new_frame;
420}
421
422// Get an openshot::Frame object for a specific frame number of this reader.
423std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
424{
425 // Check final cache, and just return the frame (if it's available)
426 std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
427 if (final_frame) return final_frame;
428
429 // Create a scoped lock, allowing only a single thread to run the following code at one time
430 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
431
432 // Find parent properties (if any)
433 Clip *parent = static_cast<Clip *>(ParentClip());
434 bool is_increasing = true;
435 if (parent) {
436 float position = parent->Position();
437 float start = parent->Start();
438 if (parent_position != position || parent_start != start) {
439 // Force dirty if parent clip has moved or been trimmed
440 // since this heavily affects frame #s and audio mappings
441 is_dirty = true;
442 }
443
444 // Determine direction of parent clip at this frame (forward or reverse direction)
445 // This is important for reversing audio in our resampler, for smooth reversed audio.
446 is_increasing = parent->time.IsIncreasing(requested_frame);
447 }
448
449 // Check if mappings are dirty (and need to be recalculated)
450 if (is_dirty)
451 Init();
452
453 // Check final cache a 2nd time (due to potential lock already generating this frame)
454 final_frame = final_cache.GetFrame(requested_frame);
455 if (final_frame) return final_frame;
456
457 // Minimum number of frames to process (for performance reasons)
458 // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
459 int minimum_frames = 1;
460
461 // Debug output
463 "FrameMapper::GetFrame (Loop through frames)",
464 "requested_frame", requested_frame,
465 "minimum_frames", minimum_frames);
466
467 // Loop through all requested frames
468 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
469 {
470 // Debug output
472 "FrameMapper::GetFrame (inside omp for loop)",
473 "frame_number", frame_number,
474 "minimum_frames", minimum_frames,
475 "requested_frame", requested_frame);
476
477 // Get the mapped frame
478 MappedFrame mapped = GetMappedFrame(frame_number);
479 std::shared_ptr<Frame> mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
480
481 // Get # of channels in the actual frame
482 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
483 int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
484
485 // Determine if mapped frame is identical to source frame
486 // including audio sample distribution according to mapped.Samples,
487 // and frame_number. In some cases such as end of stream, the reader
488 // will return a frame with a different frame number. In these cases,
489 // we cannot use the frame as is, nor can we modify the frame number,
490 // otherwise the reader's cache object internals become invalid.
491 if (info.sample_rate == mapped_frame->SampleRate() &&
492 info.channels == mapped_frame->GetAudioChannelsCount() &&
493 info.channel_layout == mapped_frame->ChannelsLayout() &&
494 mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
495 mapped.Samples.total == samples_in_frame && is_increasing &&
496 mapped.Samples.frame_start == mapped.Odd.Frame &&
497 mapped.Samples.sample_start == 0 &&
498 mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
499 info.fps.num == reader->info.fps.num &&
500 info.fps.den == reader->info.fps.den) {
501 // Add original frame to cache, and skip the rest (for performance reasons)
502 final_cache.Add(mapped_frame);
503 continue;
504 }
505
506 // Create a new frame
507 auto frame = std::make_shared<Frame>(
508 frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
509 frame->SampleRate(mapped_frame->SampleRate());
510 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
511
512
513 // Copy the image from the odd field
514 std::shared_ptr<Frame> odd_frame = mapped_frame;
515
516 if (odd_frame)
517 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()), true);
518 if (mapped.Odd.Frame != mapped.Even.Frame) {
519 // Add even lines (if different than the previous image)
520 std::shared_ptr<Frame> even_frame;
521 even_frame = GetOrCreateFrame(mapped.Even.Frame);
522 if (even_frame)
523 frame->AddImage(std::make_shared<QImage>(*even_frame->GetImage()), false);
524 }
525
526 // Determine if reader contains audio samples
527 bool reader_has_audio = frame->SampleRate() > 0 && frame->GetAudioChannelsCount() > 0;
528
529 // Resample audio on frame (if needed)
530 bool need_resampling = false;
531 if ((info.has_audio && reader_has_audio) &&
532 (info.sample_rate != frame->SampleRate() ||
533 info.channels != frame->GetAudioChannelsCount() ||
534 info.channel_layout != frame->ChannelsLayout()))
535 // Resample audio and correct # of channels if needed
536 need_resampling = true;
537
538 // create a copy of mapped.Samples that will be used by copy loop
539 SampleRange copy_samples = mapped.Samples;
540
541 if (need_resampling)
542 {
543 // Check for non-adjacent frame requests - so the resampler can be reset
544 if (abs(frame->number - previous_frame) > 1) {
545 if (avr) {
546 // Delete resampler (if exists)
547 SWR_CLOSE(avr);
548 SWR_FREE(&avr);
549 avr = NULL;
550 }
551 }
552
553 // Resampling needed, modify copy of SampleRange object that includes some additional input samples on
554 // first iteration, and continues the offset to ensure that the resampler is not input limited.
555 const int EXTRA_INPUT_SAMPLES = 48;
556
557 if (!avr) {
558 // This is the first iteration, and we need to extend # of samples for this frame
559 // Extend sample count range by an additional EXTRA_INPUT_SAMPLES
560 copy_samples.Extend(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
561 } else {
562 // Sample rate conversion has already been allocated on this clip, so
563 // this is not the first iteration. Shift position by EXTRA_INPUT_SAMPLES to correctly align samples
564 copy_samples.Shift(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
565 }
566 }
567
568 // Copy the samples
569 int samples_copied = 0;
570 int64_t starting_frame = copy_samples.frame_start;
571 while (info.has_audio && samples_copied < copy_samples.total)
572 {
573 // Init number of samples to copy this iteration
574 int remaining_samples = copy_samples.total - samples_copied;
575 int number_to_copy = 0;
576
577 // number of original samples on this frame
578 std::shared_ptr<Frame> original_frame = mapped_frame;
579 if (starting_frame != original_frame->number) {
580 original_frame = GetOrCreateFrame(starting_frame);
581 }
582
583 int original_samples = original_frame->GetAudioSamplesCount();
584
585 // Loop through each channel
586 for (int channel = 0; channel < channels_in_frame; channel++)
587 {
588 if (starting_frame == copy_samples.frame_start)
589 {
590 // Starting frame (take the ending samples)
591 number_to_copy = original_samples - copy_samples.sample_start;
592 if (number_to_copy > remaining_samples)
593 number_to_copy = remaining_samples;
594
595 // Add samples to new frame
596 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
597 }
598 else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
599 {
600 // Middle frame (take all samples)
601 number_to_copy = original_samples;
602 if (number_to_copy > remaining_samples)
603 number_to_copy = remaining_samples;
604
605 // Add samples to new frame
606 frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
607 }
608 else
609 {
610 // Ending frame (take the beginning samples)
611 number_to_copy = copy_samples.sample_end + 1;
612 if (number_to_copy > remaining_samples)
613 number_to_copy = remaining_samples;
614
615 // Add samples to new frame
616 frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
617 }
618 }
619
620 // increment frame
621 samples_copied += number_to_copy;
622 starting_frame++;
623 }
624
625 // Reverse audio (if needed)
626 if (!is_increasing)
627 frame->ReverseAudio();
628
629 // Resample audio on frame (if needed)
630 if (need_resampling)
631 // Resample audio and correct # of channels if needed
632 ResampleMappedAudio(frame, mapped.Odd.Frame);
633
634 // Add frame to final cache
635 final_cache.Add(frame);
636
637 } // for loop
638
639 // Return processed openshot::Frame
640 return final_cache.GetFrame(requested_frame);
641}
642
643void FrameMapper::PrintMapping(std::ostream* out)
644{
645 // Check if mappings are dirty (and need to be recalculated)
646 if (is_dirty)
647 // Recalculate mappings
648 Init();
649
650 // Loop through frame mappings
651 for (float map = 1; map <= frames.size(); map++)
652 {
653 MappedFrame frame = frames[map - 1];
654 *out << "Target frame #: " << map
655 << " mapped to original frame #:\t("
656 << frame.Odd.Frame << " odd, "
657 << frame.Even.Frame << " even)" << std::endl;
658
659 *out << " - Audio samples mapped to frame "
660 << frame.Samples.frame_start << ":"
661 << frame.Samples.sample_start << " to frame "
662 << frame.Samples.frame_end << ":"
663 << frame.Samples.sample_end << endl;
664 }
665
666}
667
668// Determine if reader is open or closed
670 if (reader)
671 return reader->IsOpen();
672 else
673 return false;
674}
675
676// Open the internal reader
678{
679 if (reader)
680 {
681 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
682
683 // Open the reader
684 reader->Open();
685 }
686}
687
688// Close the internal reader
690{
691 if (reader)
692 {
693 // Create a scoped lock, allowing only a single thread to run the following code at one time
694 const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
695
696 ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
697
698 // Close internal reader
699 reader->Close();
700 }
701
702 // Clear the fields & frames lists
703 Clear();
704
705 // Mark as dirty
706 is_dirty = true;
707
708 // Clear cache
709 final_cache.Clear();
710
711 // Deallocate resample buffer
712 if (avr) {
713 SWR_CLOSE(avr);
714 SWR_FREE(&avr);
715 avr = NULL;
716 }
717}
718
719
720// Generate JSON string of this object
721std::string FrameMapper::Json() const {
722
723 // Return formatted string
724 return JsonValue().toStyledString();
725}
726
727// Generate Json::Value for this object
728Json::Value FrameMapper::JsonValue() const {
729
730 // Create root json object
731 Json::Value root = ReaderBase::JsonValue(); // get parent properties
732 root["type"] = "FrameMapper";
733 if (reader) {
734 root["reader"] = reader->JsonValue();
735 }
736
737 // return JsonValue
738 return root;
739}
740
741// Load JSON string into this object
742void FrameMapper::SetJson(const std::string value) {
743
744 // Parse JSON string into JSON objects
745 try
746 {
747 const Json::Value root = openshot::stringToJson(value);
748 // Set all values that match
749 SetJsonValue(root);
750
751 if (!root["reader"].isNull()) // does Json contain a reader?
752 {
753 // Placeholder to load reader
754 // TODO: need a createReader method for this and Clip JSON methods
755 }
756 }
757 catch (const std::exception& e)
758 {
759 // Error parsing JSON (or missing keys)
760 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
761 }
762}
763
764// Load Json::Value into this object
765void FrameMapper::SetJsonValue(const Json::Value root) {
766
767 // Set parent data
769
770 // Re-Open path, and re-init everything (if needed)
771 if (reader) {
772
773 Close();
774 Open();
775 }
776}
777
778// Change frame rate or audio mapping details
779void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
780{
782 "FrameMapper::ChangeMapping",
783 "target_fps.num", target_fps.num,
784 "target_fps.den", target_fps.den,
785 "target_pulldown", target_pulldown,
786 "target_sample_rate", target_sample_rate,
787 "target_channels", target_channels,
788 "target_channel_layout", target_channel_layout);
789
790 // Mark as dirty
791 is_dirty = true;
792
793 // Update mapping details
794 target.num = target_fps.num;
795 target.den = target_fps.den;
796 info.fps.num = target_fps.num;
797 info.fps.den = target_fps.den;
798 info.video_timebase.num = target_fps.den;
799 info.video_timebase.den = target_fps.num;
801 pulldown = target_pulldown;
802 info.sample_rate = target_sample_rate;
803 info.channels = target_channels;
804 info.channel_layout = target_channel_layout;
805
806 // Enable/Disable audio (based on settings)
808
809 // Clear cache
810 final_cache.Clear();
811
812 // Adjust cache size based on size of frame and audio
814
815 // Deallocate resample buffer
816 if (avr) {
817 SWR_CLOSE(avr);
818 SWR_FREE(&avr);
819 avr = NULL;
820 }
821}
822
823// Resample audio and map channels (if needed)
824void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
825{
826 // Check if mappings are dirty (and need to be recalculated)
827 if (is_dirty)
828 // Recalculate mappings
829 Init();
830
831 // Init audio buffers / variables
832 int total_frame_samples = 0;
833 int channels_in_frame = frame->GetAudioChannelsCount();
834 int sample_rate_in_frame = frame->SampleRate();
835 int samples_in_frame = frame->GetAudioSamplesCount();
836 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
837
839 "FrameMapper::ResampleMappedAudio",
840 "frame->number", frame->number,
841 "original_frame_number", original_frame_number,
842 "channels_in_frame", channels_in_frame,
843 "samples_in_frame", samples_in_frame,
844 "sample_rate_in_frame", sample_rate_in_frame);
845
846 // Get audio sample array
847 float* frame_samples_float = NULL;
848 // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
849 frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
850
851 // Calculate total samples
852 total_frame_samples = samples_in_frame * channels_in_frame;
853
854 // Create a new array (to hold all S16 audio samples for the current queued frames)
855 int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
856
857 // Translate audio sample values back to 16 bit integers with saturation
858 float valF;
859 int16_t conv;
860 const int16_t max16 = 32767;
861 const int16_t min16 = -32768;
862 for (int s = 0; s < total_frame_samples; s++) {
863 valF = frame_samples_float[s] * (1 << 15);
864 if (valF > max16)
865 conv = max16;
866 else if (valF < min16)
867 conv = min16;
868 else
869 conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
870
871 // Copy into buffer
872 frame_samples[s] = conv;
873 }
874
875 // Deallocate float array
876 delete[] frame_samples_float;
877 frame_samples_float = NULL;
878
879 // Create input frame (and allocate arrays)
880 AVFrame *audio_frame = AV_ALLOCATE_FRAME();
881 AV_RESET_FRAME(audio_frame);
882 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
883
884 int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
885 int error_code = avcodec_fill_audio_frame(
886 audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
887 (uint8_t *) frame_samples, buf_size, 1);
888
889 if (error_code < 0)
890 {
892 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) + "]",
893 "error_code", error_code);
894 throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
895 }
896
897 // Update total samples & input frame size (due to bigger or smaller data types)
898 total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number), target, info.sample_rate, info.channels);
899
900 // Create output frame (and allocate arrays)
901 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
902 AV_RESET_FRAME(audio_converted);
903 audio_converted->nb_samples = total_frame_samples;
904 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
905
906 int nb_samples = 0;
907
908 // setup resample context
909 if (!avr) {
910 avr = SWR_ALLOC();
911 av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
912 av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
913 av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
914 av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
915 av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
916 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
917 av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
918 av_opt_set_int(avr, "out_channels", info.channels, 0);
919 SWR_INIT(avr);
920 }
921
922 // Convert audio samples
923 nb_samples = SWR_CONVERT(avr, // audio resample context
924 audio_converted->data, // output data pointers
925 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
926 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
927 audio_frame->data, // input data pointers
928 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
929 audio_frame->nb_samples); // number of input samples to convert
930
931 // Create a new array (to hold all resampled S16 audio samples)
932 int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
933
934 // Copy audio samples over original samples
935 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
936
937 // Free frames
938 av_freep(&audio_frame->data[0]);
939 AV_FREE_FRAME(&audio_frame);
940 av_freep(&audio_converted->data[0]);
941 AV_FREE_FRAME(&audio_converted);
942 frame_samples = NULL;
943
944 // Resize the frame to hold the right # of channels and samples
945 int channel_buffer_size = nb_samples;
946 frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
947
949 "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
950 "nb_samples", nb_samples,
951 "total_frame_samples", total_frame_samples,
952 "info.sample_rate", info.sample_rate,
953 "channels_in_frame", channels_in_frame,
954 "info.channels", info.channels,
955 "info.channel_layout", info.channel_layout);
956
957 // Array of floats (to hold samples for each channel)
958 float *channel_buffer = new float[channel_buffer_size];
959
960 // Divide audio into channels. Loop through each channel
961 for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
962 {
963 // Init array
964 for (int z = 0; z < channel_buffer_size; z++)
965 channel_buffer[z] = 0.0f;
966
967 // Loop through all samples and add them to our Frame based on channel.
968 // Toggle through each channel number, since channel data is stored like (left right left right)
969 int channel = 0;
970 int position = 0;
971 for (int sample = 0; sample < (nb_samples * info.channels); sample++)
972 {
973 // Only add samples for current channel
974 if (channel_filter == channel)
975 {
976 // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
977 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
978
979 // Increment audio position
980 position++;
981 }
982
983 // increment channel (if needed)
984 if ((channel + 1) < info.channels)
985 // move to next channel
986 channel ++;
987 else
988 // reset channel
989 channel = 0;
990 }
991
992 // Add samples to frame for this channel
993 frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
994 }
995
996 // Update frame's audio meta data
997 frame->SampleRate(info.sample_rate);
998 frame->ChannelsLayout(info.channel_layout);
999
1000 // clear channel buffer
1001 delete[] channel_buffer;
1002 channel_buffer = NULL;
1003
1004 // Delete arrays
1005 delete[] resampled_samples;
1006 resampled_samples = NULL;
1007
1008 // Keep track of last resampled frame
1009 previous_frame = frame->number;
1010}
1011
1012// Adjust frame number for Clip position and start (which can result in a different number)
1013int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1014
1015 // Get clip position from parent clip (if any)
1016 float position = 0.0;
1017 float start = 0.0;
1018 Clip *parent = static_cast<Clip *>(ParentClip());
1019 if (parent) {
1020 position = parent->Position();
1021 start = parent->Start();
1022 }
1023
1024 // Adjust start frame and position based on parent clip.
1025 // This ensures the same frame # is used by mapped readers and clips,
1026 // when calculating samples per frame.
1027 // Thus, this prevents gaps and mismatches in # of samples.
1028 int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1;
1029 int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1;
1030 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1031
1032 return frame_number;
1033}
Header file for Clip class.
Header file for all Exception classes.
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_ALLOCATE_FRAME()
#define SWR_FREE(ctx)
#define AV_RESET_FRAME(av_frame)
Header file for the FrameMapper class.
#define OPEN_MP_NUM_PROCESSORS
Header file for ZeroMQ-based Logger class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:30
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheMemory.cpp:46
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheMemory.cpp:80
void Clear()
Clear the cache of all frames.
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:88
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:86
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:91
openshot::Keyframe time
Curve representing the frames over time to play (used for speed and direction of video)
Definition: Clip.h:314
Exception when encoding audio packet.
Definition: Exceptions.h:143
This class represents a fraction.
Definition: Fraction.h:30
int num
Numerator for the fraction.
Definition: Fraction.h:32
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:35
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:45
int den
Denominator for the fraction.
Definition: Fraction.h:33
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool IsOpen() override
Determine if reader is open or closed.
std::vector< Field > fields
Definition: FrameMapper.h:239
ReaderBase * Reader()
Get the current reader.
Definition: FrameMapper.cpp:64
std::vector< MappedFrame > frames
Definition: FrameMapper.h:240
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Open() override
Open the internal reader.
void Close() override
Close the openshot::FrameMapper and internal reader.
std::string Json() const override
Generate JSON string of this object.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
void SetJson(const std::string value) override
Load JSON string into this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:55
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:480
Exception for invalid JSON.
Definition: Exceptions.h:218
bool IsIncreasing(int index) const
Get the direction of the curve at a specific index (increasing or decreasing)
Definition: KeyFrame.cpp:292
Exception for frames that are out of bounds.
Definition: Exceptions.h:301
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:76
virtual bool IsOpen()=0
Determine if reader is open or closed.
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:88
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:162
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:107
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
Definition: ReaderBase.h:79
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:245
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:364
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:43
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:44
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:45
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:46
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds a single field (half a frame).
Definition: FrameMapper.h:56
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:173
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:42
float duration
Length of time (in seconds)
Definition: ReaderBase.h:43
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:46
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:61
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:48
int height
The height of the video (in pixels)
Definition: ReaderBase.h:45
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:53
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:62
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:40
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:41
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:55
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:60
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:76
void Extend(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)
Extend SampleRange on either side.
Definition: FrameMapper.h:84
void Shift(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)
Definition: FrameMapper.h:148