26 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0), parent_start(0.0), previous_frame(0)
70 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
73void FrameMapper::AddField(int64_t frame)
76 AddField(
Field(frame, field_toggle));
79void FrameMapper::AddField(
Field field)
85 field_toggle = (field_toggle ? false :
true);
89void FrameMapper::Clear() {
91 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
104void FrameMapper::Init()
114 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
122 parent_position = parent->
Position();
123 parent_start = parent->
Start();
125 parent_position = 0.0;
137 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
138 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
141 float difference = target.
ToInt() - original.
ToInt();
144 int field_interval = 0;
145 int frame_interval = 0;
149 field_interval = round(fabs(original.
ToInt() / difference));
152 frame_interval = field_interval * 2.0f;
161 for (int64_t field = 1; field <= number_of_fields; field++)
169 else if (difference > 0)
179 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
186 AddField(
Field(frame + 1, field_toggle));
188 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
195 else if (difference < 0)
201 field_toggle = (field_toggle ? false :
true);
203 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
208 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
221 if (field % 2 == 0 && field > 0)
234 double original_frame_num = 1.0f;
235 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
238 AddField(round(original_frame_num));
239 AddField(round(original_frame_num));
242 original_frame_num += value_increment;
251 int64_t start_samples_frame = 1;
252 int start_samples_position = 0;
254 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
260 if (field % 2 == 0 && field > 0)
263 int64_t frame_number = field / 2;
274 int64_t end_samples_frame = start_samples_frame;
275 int end_samples_position = start_samples_position;
278 while (remaining_samples > 0)
285 if (original_samples >= remaining_samples)
288 end_samples_position += remaining_samples - 1;
289 remaining_samples = 0;
293 end_samples_frame += 1;
294 end_samples_position = 0;
295 remaining_samples -= original_samples;
305 start_samples_frame = end_samples_frame;
306 start_samples_position = end_samples_position + 1;
309 start_samples_frame += 1;
310 start_samples_position = 0;
351 frame.
Odd.
Frame = TargetFrameNumber;
361 if(TargetFrameNumber < 1 ||
frames.size() == 0)
365 else if (TargetFrameNumber > (int64_t)
frames.size())
367 TargetFrameNumber =
frames.size();
371 "FrameMapper::GetMappedFrame",
372 "TargetFrameNumber", TargetFrameNumber,
373 "frames.size()",
frames.size(),
374 "frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
375 "frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
378 return frames[TargetFrameNumber - 1];
382std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
384 std::shared_ptr<Frame> new_frame;
392 "FrameMapper::GetOrCreateFrame (from reader)",
394 "samples_in_frame", samples_in_frame);
397 new_frame = reader->
GetFrame(number);
410 "FrameMapper::GetOrCreateFrame (create blank)",
412 "samples_in_frame", samples_in_frame);
418 new_frame->AddAudioSilence(samples_in_frame);
426 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
427 if (final_frame)
return final_frame;
430 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
434 bool is_increasing =
true;
436 float position = parent->
Position();
437 float start = parent->
Start();
438 if (parent_position != position || parent_start != start) {
454 final_frame = final_cache.
GetFrame(requested_frame);
455 if (final_frame)
return final_frame;
459 int minimum_frames = 1;
463 "FrameMapper::GetFrame (Loop through frames)",
464 "requested_frame", requested_frame,
465 "minimum_frames", minimum_frames);
468 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
472 "FrameMapper::GetFrame (inside omp for loop)",
473 "frame_number", frame_number,
474 "minimum_frames", minimum_frames,
475 "requested_frame", requested_frame);
479 std::shared_ptr<Frame> mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
482 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
483 int samples_in_frame =
Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
492 info.
channels == mapped_frame->GetAudioChannelsCount() &&
494 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
495 mapped.
Samples.
total == samples_in_frame && is_increasing &&
498 mapped_frame->number == frame_number &&
502 final_cache.
Add(mapped_frame);
507 auto frame = std::make_shared<Frame>(
508 frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
509 frame->SampleRate(mapped_frame->SampleRate());
510 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
514 std::shared_ptr<Frame> odd_frame = mapped_frame;
517 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()),
true);
520 std::shared_ptr<Frame> even_frame;
521 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
523 frame->AddImage(std::make_shared<QImage>(*even_frame->GetImage()),
false);
527 bool reader_has_audio = frame->SampleRate() > 0 && frame->GetAudioChannelsCount() > 0;
530 bool need_resampling =
false;
536 need_resampling =
true;
544 if (abs(frame->number - previous_frame) > 1) {
555 const int EXTRA_INPUT_SAMPLES = 48;
569 int samples_copied = 0;
574 int remaining_samples = copy_samples.
total - samples_copied;
575 int number_to_copy = 0;
578 std::shared_ptr<Frame> original_frame = mapped_frame;
579 if (starting_frame != original_frame->number) {
580 original_frame = GetOrCreateFrame(starting_frame);
583 int original_samples = original_frame->GetAudioSamplesCount();
586 for (
int channel = 0; channel < channels_in_frame; channel++)
591 number_to_copy = original_samples - copy_samples.
sample_start;
592 if (number_to_copy > remaining_samples)
593 number_to_copy = remaining_samples;
596 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
601 number_to_copy = original_samples;
602 if (number_to_copy > remaining_samples)
603 number_to_copy = remaining_samples;
606 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
612 if (number_to_copy > remaining_samples)
613 number_to_copy = remaining_samples;
616 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
621 samples_copied += number_to_copy;
627 frame->ReverseAudio();
635 final_cache.
Add(frame);
640 return final_cache.
GetFrame(requested_frame);
651 for (
float map = 1; map <=
frames.size(); map++)
654 *out <<
"Target frame #: " << map
655 <<
" mapped to original frame #:\t("
657 << frame.
Even.
Frame <<
" even)" << std::endl;
659 *out <<
" - Audio samples mapped to frame "
694 const std::lock_guard<std::recursive_mutex> lock(
getFrameMutex);
732 root[
"type"] =
"FrameMapper";
751 if (!root[
"reader"].isNull())
757 catch (
const std::exception& e)
760 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
782 "FrameMapper::ChangeMapping",
783 "target_fps.num", target_fps.
num,
784 "target_fps.den", target_fps.
den,
785 "target_pulldown", target_pulldown,
786 "target_sample_rate", target_sample_rate,
787 "target_channels", target_channels,
788 "target_channel_layout", target_channel_layout);
794 target.
num = target_fps.
num;
795 target.
den = target_fps.
den;
801 pulldown = target_pulldown;
832 int total_frame_samples = 0;
833 int channels_in_frame = frame->GetAudioChannelsCount();
834 int sample_rate_in_frame = frame->SampleRate();
835 int samples_in_frame = frame->GetAudioSamplesCount();
836 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
839 "FrameMapper::ResampleMappedAudio",
840 "frame->number", frame->number,
841 "original_frame_number", original_frame_number,
842 "channels_in_frame", channels_in_frame,
843 "samples_in_frame", samples_in_frame,
844 "sample_rate_in_frame", sample_rate_in_frame);
847 float* frame_samples_float = NULL;
849 frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
852 total_frame_samples = samples_in_frame * channels_in_frame;
855 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
860 const int16_t max16 = 32767;
861 const int16_t min16 = -32768;
862 for (
int s = 0; s < total_frame_samples; s++) {
863 valF = frame_samples_float[s] * (1 << 15);
866 else if (valF < min16)
869 conv = int(valF + 32768.5) - 32768;
872 frame_samples[s] = conv;
876 delete[] frame_samples_float;
877 frame_samples_float = NULL;
882 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
884 int buf_size = audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame;
885 int error_code = avcodec_fill_audio_frame(
886 audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16,
887 (uint8_t *) frame_samples, buf_size, 1);
892 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) +
"]",
893 "error_code", error_code);
894 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
903 audio_converted->nb_samples = total_frame_samples;
904 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
911 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
913 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
914 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
915 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
917 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
924 audio_converted->data,
925 audio_converted->linesize[0],
926 audio_converted->nb_samples,
928 audio_frame->linesize[0],
929 audio_frame->nb_samples);
932 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
935 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
938 av_freep(&audio_frame->data[0]);
940 av_freep(&audio_converted->data[0]);
942 frame_samples = NULL;
945 int channel_buffer_size = nb_samples;
949 "FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
950 "nb_samples", nb_samples,
951 "total_frame_samples", total_frame_samples,
953 "channels_in_frame", channels_in_frame,
958 float *channel_buffer =
new float[channel_buffer_size];
961 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
964 for (
int z = 0; z < channel_buffer_size; z++)
965 channel_buffer[z] = 0.0f;
971 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
974 if (channel_filter == channel)
977 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
993 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
1001 delete[] channel_buffer;
1002 channel_buffer = NULL;
1005 delete[] resampled_samples;
1006 resampled_samples = NULL;
1009 previous_frame = frame->number;
1013int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
1016 float position = 0.0;
1021 start = parent->
Start();
1029 int64_t clip_start_position = round(position *
info.
fps.
ToDouble()) + 1;
1030 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
1032 return frame_number;
Header file for Clip class.
Header file for all Exception classes.
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define AV_ALLOCATE_FRAME()
#define AV_RESET_FRAME(av_frame)
Header file for the FrameMapper class.
#define OPEN_MP_NUM_PROCESSORS
Header file for ZeroMQ-based Logger class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Position() const
Get position on timeline (in seconds)
This class represents a clip (used to arrange readers on the timeline)
openshot::Keyframe time
Curve representing the frames over time to play (used for speed and direction of video)
Exception when encoding audio packet.
This class represents a fraction.
int num
Numerator for the fraction.
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
int den
Denominator for the fraction.
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool IsOpen() override
Determine if reader is open or closed.
std::vector< Field > fields
ReaderBase * Reader()
Get the current reader.
std::vector< MappedFrame > frames
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Open() override
Open the internal reader.
void Close() override
Close the openshot::FrameMapper and internal reader.
std::string Json() const override
Generate JSON string of this object.
void PrintMapping(std::ostream *out=&std::cout)
Print all of the original frames and which new frames they map to.
void SetJson(const std::string value) override
Load JSON string into this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Exception for invalid JSON.
bool IsIncreasing(int index) const
Get the direction of the curve at a specific index (increasing or decreasing)
Exception for frames that are out of bounds.
This abstract class is the base class, used by all readers in libopenshot.
virtual bool IsOpen()=0
Determine if reader is open or closed.
openshot::ReaderInfo info
Information about the current media file.
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
std::recursive_mutex getFrameMutex
Mutex for multiple threads.
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
This namespace is the default namespace for all code in the openshot library.
PulldownType
This enumeration determines how frame rates are increased or decreased.
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
This struct holds a single field (half a frame).
This struct holds two fields which together make up a complete video frame.
bool has_single_image
Determines if this file only contains a single image.
float duration
Length of time (in seconds)
int width
The width of the video (in pixesl)
int channels
The number of audio channels used in the audio stream.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
int height
The height of the video (in pixels)
int64_t video_length
The number of frames in the video stream.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
bool has_video
Determines if this file has a video stream.
bool has_audio
Determines if this file has an audio stream.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
This struct holds a the range of samples needed by this frame.
void Extend(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)
Extend SampleRange on either side.
void Shift(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side)