21#include "objdetectdata.pb.h"
31ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
34 init_effect_details();
47 init_effect_details();
54void ObjectDetection::init_effect_details()
73 cv::Mat cv_image = frame->GetImageCV();
81 std::vector<QRectF> boxRects;
83 std::vector<std::shared_ptr<QImage>> childClipImages;
86 if (detectionsData.find(frame_number) != detectionsData.end()) {
87 float fw = cv_image.size().width;
88 float fh = cv_image.size().height;
91 for(
int i = 0; i<detections.
boxes.size(); i++){
94 if(detections.
confidences.at(i) < confidence_threshold){
98 if( display_classes.size() > 0 &&
99 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
104 int objectId = detections.
objectIds.at(i);
110 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
113 if (trackedObject->Contains(frame_number) &&
114 trackedObject->visible.GetValue(frame_number) == 1)
117 BBox trackedBox = trackedObject->GetBox(frame_number);
118 bool draw_text = !display_box_text.
GetValue(frame_number);
119 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
120 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
121 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
122 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
123 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
135 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
136 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
137 (
int)( trackedBox.
width*fw),
138 (
int)( trackedBox.
height*fh)
142 if (trackedObject->draw_box.GetValue(frame_number) == 0)
149 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
151 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
155 if (trackedObject->ChildClipId() !=
""){
160 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
163 std::shared_ptr<Frame> f(
new Frame(1, frame->GetWidth(), frame->GetHeight(),
"#00000000"));
165 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(f, frame_number);
166 childClipImages.push_back(childClipFrame->GetImage());
170 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
171 (
int)((trackedBox.
cy - trackedBox.
height/2)*fh),
172 (
int)(trackedBox.
width*fw),
173 (
int)(trackedBox.
height*fh));
174 boxRects.push_back(boxRect);
183 frame->SetImageCV(cv_image);
186 if(boxRects.size() > 0){
188 QImage frameImage = *(frame->GetImage());
189 for(
int i; i < boxRects.size();i++){
191 QPainter painter(&frameImage);
193 painter.drawImage(boxRects[i], *childClipImages[i], QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
196 frame->AddImage(std::make_shared<QImage>(frameImage));
202void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
203 int thickness,
bool is_background){
205 cv::Point2f vertices2f[4];
206 box.points(vertices2f);
214 cv::Mat overlayFrame;
215 frame_image.copyTo(overlayFrame);
218 cv::Point vertices[4];
219 for(
int i = 0; i < 4; ++i){
220 vertices[i] = vertices2f[i];}
222 cv::Rect rect = box.boundingRect();
223 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
225 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
228 cv::Mat overlayFrame;
229 frame_image.copyTo(overlayFrame);
232 for (
int i = 0; i < 4; i++)
234 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
235 thickness, cv::LINE_AA);
239 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
243void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
244 float alpha,
int thickness,
bool is_background,
bool display_text)
248 cv::Mat overlayFrame;
249 frame.copyTo(overlayFrame);
252 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
255 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
258 cv::Mat overlayFrame;
259 frame.copyTo(overlayFrame);
262 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
266 std::string label = cv::format(
"%.2f", conf);
267 if (!classNames.empty())
269 CV_Assert(classId < (
int)classNames.size());
270 label = classNames[classId] +
":" + label;
275 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
278 double top = std::max((
int)box.y, labelSize.height);
280 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
281 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
282 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
285 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
292 pb_objdetect::ObjDetect objMessage;
295 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
296 if (!objMessage.ParseFromIstream(&input)) {
297 std::cerr <<
"Failed to parse protobuf message." << std::endl;
303 detectionsData.clear();
309 for(
int i = 0; i < objMessage.classnames_size(); i++)
311 classNames.push_back(objMessage.classnames(i));
312 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
316 for (
size_t i = 0; i < objMessage.frame_size(); i++)
319 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
322 size_t id = pbFrameData.id();
325 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
328 std::vector<int> classIds;
329 std::vector<float> confidences;
330 std::vector<cv::Rect_<float>> boxes;
331 std::vector<int> objectIds;
334 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
337 float x = pBox.Get(i).x();
338 float y = pBox.Get(i).y();
339 float w = pBox.Get(i).w();
340 float h = pBox.Get(i).h();
342 int classId = pBox.Get(i).classid();
344 float confidence = pBox.Get(i).confidence();
347 int objectId = pBox.Get(i).objectid();
355 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
360 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
361 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
363 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
365 trackedObjPtr->ParentClip(parentClip);
369 trackedObjPtr->Id(std::to_string(objectId));
374 cv::Rect_<float> box(x, y, w, h);
377 boxes.push_back(box);
378 classIds.push_back(classId);
379 confidences.push_back(confidence);
380 objectIds.push_back(objectId);
384 detectionsData[
id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
388 google::protobuf::ShutdownProtobufLibrary();
398 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
399 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
402 if (detectionsData.find(frame_number) == detectionsData.end()){
403 return root.toStyledString();
408 for(
int i = 0; i<detections.
boxes.size(); i++){
410 if(detections.
confidences.at(i) < confidence_threshold){
415 if( display_classes.size() > 0 &&
416 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
420 int objectId = detections.
objectIds.at(i);
425 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
427 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
428 trackedObject->second->ExactlyContains(frame_number)){
430 root[
"visible_objects_index"].append(trackedObject->first);
431 root[
"visible_objects_id"].append(trackedObject->second->Id());
435 return root.toStyledString();
451 root[
"protobuf_data_path"] = protobuf_data_path;
453 root[
"confidence_threshold"] = confidence_threshold;
454 root[
"display_box_text"] = display_box_text.
JsonValue();
459 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
461 objects[trackedObject.second->Id()] = trackedObjectJSON;
463 root[
"objects"] = objects;
479 catch (
const std::exception& e)
482 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
492 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
493 protobuf_data_path = root[
"protobuf_data_path"].asString();
496 throw InvalidFile(
"Invalid protobuf data path",
"");
497 protobuf_data_path =
"";
502 if (!root[
"selected_object_index"].isNull())
505 if (!root[
"confidence_threshold"].isNull())
506 confidence_threshold = root[
"confidence_threshold"].asFloat();
508 if (!root[
"display_box_text"].isNull())
509 display_box_text.
SetJsonValue(root[
"display_box_text"]);
511 if (!root[
"class_filter"].isNull()){
512 class_filter = root[
"class_filter"].asString();
513 std::stringstream ss(class_filter);
514 display_classes.clear();
519 std::getline( ss, substr,
',' );
520 display_classes.push_back( substr );
524 if (!root[
"objects"].isNull()){
526 std::string obj_id = std::to_string(trackedObject.first);
527 if(!root[
"objects"][obj_id].isNull()){
528 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
534 if (!root[
"objects_id"].isNull()){
536 Json::Value trackedObjectJSON;
537 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
538 trackedObject.second->SetJsonValue(trackedObjectJSON);
553 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
555 objects[selectedObject->Id()] = trackedObjectJSON;
558 root[
"objects"] = objects;
561 root[
"id"] =
add_property_json(
"ID", 0.0,
"string",
Id(), NULL, -1, -1,
true, requested_frame);
562 root[
"position"] =
add_property_json(
"Position",
Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
564 root[
"start"] =
add_property_json(
"Start",
Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
565 root[
"end"] =
add_property_json(
"End",
End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
566 root[
"duration"] =
add_property_json(
"Duration",
Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
567 root[
"confidence_threshold"] =
add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
568 root[
"class_filter"] =
add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
570 root[
"display_box_text"] =
add_property_json(
"Draw Box Text", display_box_text.
GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
575 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Duration() const
Get the length of this clip (in seconds)
virtual float End() const
Get end position (in seconds) of clip (trim end of video)
std::string Id() const
Get the Id of this clip object.
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
float Position() const
Get position on timeline (in seconds)
virtual openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
std::string id
ID Property for all derived Clip and Effect classes.
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
This class represents a clip (used to arrange readers on the timeline)
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
EffectInfoStruct info
Information about the current effect.
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
This class represents a single frame of video (i.e. image & audio data)
Exception for files that can not be found or opened.
Exception for invalid JSON.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
double GetValue(int64_t index) const
Get the value at a specific index.
Json::Value JsonValue() const
Generate Json::Value for this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
int selectedObjectIndex
Index of the Tracked Object that was selected to modify it's properties.
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
This method is required for all derived classes of EffectBase, and returns a modified openshot::Frame...
ObjectDetection()
Default constructor.
bool LoadObjDetectdData(std::string inputFilePath)
Load protobuf data file.
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
std::string Json() const override
Generate JSON string of this object.
std::string PropertiesJSON(int64_t requested_frame) const override
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a timeline.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
bool has_video
Determines if this effect manipulates the image of a frame.
bool has_audio
Determines if this effect manipulates the audio of a frame.
std::string class_name
The class name of the effect.
std::string name
The name of the effect.
std::string description
The description of this effect and what it does.
bool has_tracked_object
Determines if this effect track objects through the clip.