OpenShot Library | libopenshot 0.3.1
Tracker.cpp
Go to the documentation of this file.
1
10// Copyright (c) 2008-2019 OpenShot Studios, LLC
11//
12// SPDX-License-Identifier: LGPL-3.0-or-later
13
14#include <string>
15#include <memory>
16#include <fstream>
17#include <iostream>
18
19#include "effects/Tracker.h"
20#include "Exceptions.h"
21#include "Timeline.h"
22#include "trackerdata.pb.h"
23
24#include <google/protobuf/util/time_util.h>
25
26#include <QImage>
27#include <QPainter>
28#include <QRectF>
29
30using namespace std;
31using namespace openshot;
32using google::protobuf::util::TimeUtil;
33
35Tracker::Tracker(std::string clipTrackerDataPath)
36{
37 // Init effect properties
38 init_effect_details();
39 // Instantiate a TrackedObjectBBox object and point to it
40 TrackedObjectBBox trackedDataObject;
41 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
42 // Tries to load the tracked object's data from protobuf file
43 trackedData->LoadBoxData(clipTrackerDataPath);
44 ClipBase* parentClip = this->ParentClip();
45 trackedData->ParentClip(parentClip);
46 trackedData->Id(std::to_string(0));
47 // Insert TrackedObject with index 0 to the trackedObjects map
48 trackedObjects.insert({0, trackedData});
49}
50
51// Default constructor
53{
54 // Init effect properties
55 init_effect_details();
56 // Instantiate a TrackedObjectBBox object and point to it
57 TrackedObjectBBox trackedDataObject;
58 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59 ClipBase* parentClip = this->ParentClip();
60 trackedData->ParentClip(parentClip);
61 trackedData->Id(std::to_string(0));
62 // Insert TrackedObject with index 0 to the trackedObjects map
63 trackedObjects.insert({0, trackedData});
64}
65
66
67// Init effect settings
68void Tracker::init_effect_details()
69{
72
74 info.class_name = "Tracker";
75 info.name = "Tracker";
76 info.description = "Track the selected bounding box through the video.";
77 info.has_audio = false;
78 info.has_video = true;
80
81 this->TimeScale = 1.0;
82}
83
84// This method is required for all derived classes of EffectBase, and returns a
85// modified openshot::Frame object
86std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
87{
88 // Get the frame's image
89 cv::Mat frame_image = frame->GetImageCV();
90
91 // Initialize the Qt rectangle that will hold the positions of the bounding-box
92 QRectF boxRect;
93 // Initialize the image of the TrackedObject child clip
94 std::shared_ptr<QImage> childClipImage = nullptr;
95
96 // Check if frame isn't NULL
97 if(!frame_image.empty() &&
98 trackedData->Contains(frame_number) &&
99 trackedData->visible.GetValue(frame_number) == 1)
100 {
101 // Get the width and height of the image
102 float fw = frame_image.size().width;
103 float fh = frame_image.size().height;
104
105 // Get the bounding-box of given frame
106 BBox fd = trackedData->GetBox(frame_number);
107
108 // Check if track data exists for the requested frame
109 if (trackedData->draw_box.GetValue(frame_number) == 1)
110 {
111 std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
112 int stroke_width = trackedData->stroke_width.GetValue(frame_number);
113 float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
114 std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
115 float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
116
117 // Create a rotated rectangle object that holds the bounding box
118 cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
119 cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
120 (int) (fd.angle) );
121
122 DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
123 DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
124 }
125
126 // Get the image of the Tracked Object' child clip
127 if (trackedData->ChildClipId() != ""){
128 // Cast the parent timeline of this effect
129 Timeline* parentTimeline = static_cast<Timeline *>(ParentTimeline());
130 if (parentTimeline){
131 // Get the Tracked Object's child clip
132 Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
133 if (childClip){
134 // Get the image of the child clip for this frame
135 std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
136 std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
137 childClipImage = childClipFrame->GetImage();
138
139 // Set the Qt rectangle with the bounding-box properties
140 boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
141 (int)((fd.cy - fd.height/2)*fh),
142 (int)(fd.width*fw),
143 (int)(fd.height*fh) );
144 }
145 }
146 }
147
148 }
149
150 // Set image with drawn box to frame
151 // If the input image is NULL or doesn't have tracking data, it's returned as it came
152 frame->SetImageCV(frame_image);
153
154 // Set the bounding-box image with the Tracked Object's child clip image
155 if (childClipImage){
156 // Get the frame image
157 QImage frameImage = *(frame->GetImage());
158
159 // Set a Qt painter to the frame image
160 QPainter painter(&frameImage);
161
162 // Draw the child clip image inside the bounding-box
163 painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
164
165 // Set the frame image as the composed image
166 frame->AddImage(std::make_shared<QImage>(frameImage));
167 }
168
169 return frame;
170}
171
172void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
173 // Get the bouding box vertices
174 cv::Point2f vertices2f[4];
175 box.points(vertices2f);
176
177 // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
178 // select min enclosing rectangle to draw on a small portion of the image
179 // cv::Rect rect = box.boundingRect();
180 // cv::Mat image = frame_image(rect)
181
182 if(is_background){
183 cv::Mat overlayFrame;
184 frame_image.copyTo(overlayFrame);
185
186 // draw bounding box background
187 cv::Point vertices[4];
188 for(int i = 0; i < 4; ++i){
189 vertices[i] = vertices2f[i];}
190
191 cv::Rect rect = box.boundingRect();
192 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
193 // add opacity
194 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
195 }
196 else{
197 cv::Mat overlayFrame;
198 frame_image.copyTo(overlayFrame);
199
200 // Draw bounding box
201 for (int i = 0; i < 4; i++)
202 {
203 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
204 thickness, cv::LINE_AA);
205 }
206
207 // add opacity
208 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
209 }
210}
211
212// Get the indexes and IDs of all visible objects in the given frame
213std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
214
215 // Initialize the JSON objects
216 Json::Value root;
217 root["visible_objects_index"] = Json::Value(Json::arrayValue);
218 root["visible_objects_id"] = Json::Value(Json::arrayValue);
219
220 // Iterate through the tracked objects
221 for (const auto& trackedObject : trackedObjects){
222 // Get the tracked object JSON properties for this frame
223 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
224 if (trackedObjectJSON["visible"]["value"].asBool()){
225 // Save the object's index and ID if it's visible in this frame
226 root["visible_objects_index"].append(trackedObject.first);
227 root["visible_objects_id"].append(trackedObject.second->Id());
228 }
229 }
230
231 return root.toStyledString();
232}
233
234// Generate JSON string of this object
235std::string Tracker::Json() const {
236
237 // Return formatted string
238 return JsonValue().toStyledString();
239}
240
241// Generate Json::Value for this object
242Json::Value Tracker::JsonValue() const {
243
244 // Create root json object
245 Json::Value root = EffectBase::JsonValue(); // get parent properties
246
247 // Save the effect's properties on root
248 root["type"] = info.class_name;
249 root["protobuf_data_path"] = protobuf_data_path;
250 root["BaseFPS"]["num"] = BaseFPS.num;
251 root["BaseFPS"]["den"] = BaseFPS.den;
252 root["TimeScale"] = this->TimeScale;
253
254 // Add trackedObjects IDs to JSON
255 Json::Value objects;
256 for (auto const& trackedObject : trackedObjects){
257 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
258 // add object json
259 objects[trackedObject.second->Id()] = trackedObjectJSON;
260 }
261 root["objects"] = objects;
262
263 // return JsonValue
264 return root;
265}
266
267// Load JSON string into this object
268void Tracker::SetJson(const std::string value) {
269
270 // Parse JSON string into JSON objects
271 try
272 {
273 const Json::Value root = openshot::stringToJson(value);
274 // Set all values that match
275 SetJsonValue(root);
276 }
277 catch (const std::exception& e)
278 {
279 // Error parsing JSON (or missing keys)
280 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
281 }
282 return;
283}
284
285// Load Json::Value into this object
286void Tracker::SetJsonValue(const Json::Value root) {
287
288 // Set parent data
290
291 if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
292 {
293 if (!root["BaseFPS"]["num"].isNull())
294 {
295 BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
296 }
297 if (!root["BaseFPS"]["den"].isNull())
298 {
299 BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
300 }
301 }
302
303 if (!root["TimeScale"].isNull())
304 TimeScale = (double) root["TimeScale"].asDouble();
305
306 // Set data from Json (if key is found)
307 if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
308 {
309 protobuf_data_path = root["protobuf_data_path"].asString();
310 if(!trackedData->LoadBoxData(protobuf_data_path))
311 {
312 std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
314 }
315 }
316
317 if (!root["objects"].isNull()){
318 for (auto const& trackedObject : trackedObjects){
319 std::string obj_id = std::to_string(trackedObject.first);
320 if(!root["objects"][obj_id].isNull()){
321 trackedObject.second->SetJsonValue(root["objects"][obj_id]);
322 }
323 }
324 }
325
326 // Set the tracked object's ids
327 if (!root["objects_id"].isNull()){
328 for (auto const& trackedObject : trackedObjects){
329 Json::Value trackedObjectJSON;
330 trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
331 trackedObject.second->SetJsonValue(trackedObjectJSON);
332 }
333 }
334
335 return;
336}
337
338// Get all properties for a specific frame
339std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
340
341 // Generate JSON properties list
342 Json::Value root;
343
344 // Add trackedObject properties to JSON
345 Json::Value objects;
346 for (auto const& trackedObject : trackedObjects){
347 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
348 // add object json
349 objects[trackedObject.second->Id()] = trackedObjectJSON;
350 }
351 root["objects"] = objects;
352
353 // Append effect's properties
354 root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
355 root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
356 root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
357 root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
358 root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
359 root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
360
361 // Return formatted string
362 return root.toStyledString();
363}
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:33
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:88
float Duration() const
Get the length of this clip (in seconds)
Definition: ClipBase.h:90
virtual float End() const
Get end position (in seconds) of clip (trim end of video)
Definition: ClipBase.h:89
std::string Id() const
Get the Id of this clip object.
Definition: ClipBase.h:85
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
Definition: ClipBase.h:87
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:86
virtual openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:91
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
Definition: ClipBase.cpp:96
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:91
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:389
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: EffectBase.cpp:77
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
Definition: EffectBase.cpp:173
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: EffectBase.cpp:112
EffectInfoStruct info
Information about the current effect.
Definition: EffectBase.h:69
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Definition: EffectBase.h:66
int num
Numerator for the fraction.
Definition: Fraction.h:32
int den
Denominator for the fraction.
Definition: Fraction.h:33
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:91
Exception for invalid JSON.
Definition: Exceptions.h:218
This class represents a timeline.
Definition: Timeline.h:150
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:408
This class contains the properties of a tracked object and functions to manipulate it.
std::string Json() const override
Generate JSON string of this object.
Definition: Tracker.cpp:235
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
Definition: Tracker.cpp:213
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: Tracker.cpp:242
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: Tracker.cpp:268
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
Apply this effect to an openshot::Frame.
Definition: Tracker.cpp:86
std::string PropertiesJSON(int64_t requested_frame) const override
Definition: Tracker.cpp:339
std::shared_ptr< TrackedObjectBBox > trackedData
Pointer to an object that holds the bounding-box data and it's Keyframes.
Definition: Tracker.h:52
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: Tracker.cpp:286
Tracker()
Default constructor.
Definition: Tracker.cpp:52
void DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector< int > color, float alpha, int thickness, bool is_background)
Definition: Tracker.cpp:172
std::string protobuf_data_path
Path to the protobuf file that holds the bounding-box data.
Definition: Tracker.h:51
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]
bool has_video
Determines if this effect manipulates the image of a frame.
Definition: EffectBase.h:40
bool has_audio
Determines if this effect manipulates the audio of a frame.
Definition: EffectBase.h:41
std::string class_name
The class name of the effect.
Definition: EffectBase.h:36
std::string name
The name of the effect.
Definition: EffectBase.h:37
std::string description
The description of this effect and what it does.
Definition: EffectBase.h:38
bool has_tracked_object
Determines if this effect track objects through the clip.
Definition: EffectBase.h:42