rtmlib-ts 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/.gitattributes +1 -0
  2. package/README.md +202 -0
  3. package/dist/core/base.d.ts +20 -0
  4. package/dist/core/base.d.ts.map +1 -0
  5. package/dist/core/base.js +40 -0
  6. package/dist/core/file.d.ts +11 -0
  7. package/dist/core/file.d.ts.map +1 -0
  8. package/dist/core/file.js +111 -0
  9. package/dist/core/modelCache.d.ts +35 -0
  10. package/dist/core/modelCache.d.ts.map +1 -0
  11. package/dist/core/modelCache.js +161 -0
  12. package/dist/core/posePostprocessing.d.ts +12 -0
  13. package/dist/core/posePostprocessing.d.ts.map +1 -0
  14. package/dist/core/posePostprocessing.js +76 -0
  15. package/dist/core/postprocessing.d.ts +10 -0
  16. package/dist/core/postprocessing.d.ts.map +1 -0
  17. package/dist/core/postprocessing.js +70 -0
  18. package/dist/core/preprocessing.d.ts +14 -0
  19. package/dist/core/preprocessing.d.ts.map +1 -0
  20. package/dist/core/preprocessing.js +79 -0
  21. package/dist/index.d.ts +27 -0
  22. package/dist/index.d.ts.map +1 -0
  23. package/dist/index.js +31 -0
  24. package/dist/models/rtmpose.d.ts +25 -0
  25. package/dist/models/rtmpose.d.ts.map +1 -0
  26. package/dist/models/rtmpose.js +185 -0
  27. package/dist/models/rtmpose3d.d.ts +28 -0
  28. package/dist/models/rtmpose3d.d.ts.map +1 -0
  29. package/dist/models/rtmpose3d.js +184 -0
  30. package/dist/models/yolo12.d.ts +23 -0
  31. package/dist/models/yolo12.d.ts.map +1 -0
  32. package/dist/models/yolo12.js +165 -0
  33. package/dist/models/yolox.d.ts +18 -0
  34. package/dist/models/yolox.d.ts.map +1 -0
  35. package/dist/models/yolox.js +167 -0
  36. package/dist/solution/animalDetector.d.ts +229 -0
  37. package/dist/solution/animalDetector.d.ts.map +1 -0
  38. package/dist/solution/animalDetector.js +663 -0
  39. package/dist/solution/body.d.ts +16 -0
  40. package/dist/solution/body.d.ts.map +1 -0
  41. package/dist/solution/body.js +52 -0
  42. package/dist/solution/bodyWithFeet.d.ts +16 -0
  43. package/dist/solution/bodyWithFeet.d.ts.map +1 -0
  44. package/dist/solution/bodyWithFeet.js +52 -0
  45. package/dist/solution/customDetector.d.ts +137 -0
  46. package/dist/solution/customDetector.d.ts.map +1 -0
  47. package/dist/solution/customDetector.js +342 -0
  48. package/dist/solution/hand.d.ts +14 -0
  49. package/dist/solution/hand.d.ts.map +1 -0
  50. package/dist/solution/hand.js +20 -0
  51. package/dist/solution/index.d.ts +10 -0
  52. package/dist/solution/index.d.ts.map +1 -0
  53. package/dist/solution/index.js +9 -0
  54. package/dist/solution/objectDetector.d.ts +172 -0
  55. package/dist/solution/objectDetector.d.ts.map +1 -0
  56. package/dist/solution/objectDetector.js +606 -0
  57. package/dist/solution/pose3dDetector.d.ts +145 -0
  58. package/dist/solution/pose3dDetector.d.ts.map +1 -0
  59. package/dist/solution/pose3dDetector.js +611 -0
  60. package/dist/solution/poseDetector.d.ts +198 -0
  61. package/dist/solution/poseDetector.d.ts.map +1 -0
  62. package/dist/solution/poseDetector.js +622 -0
  63. package/dist/solution/poseTracker.d.ts +22 -0
  64. package/dist/solution/poseTracker.d.ts.map +1 -0
  65. package/dist/solution/poseTracker.js +106 -0
  66. package/dist/solution/wholebody.d.ts +19 -0
  67. package/dist/solution/wholebody.d.ts.map +1 -0
  68. package/dist/solution/wholebody.js +82 -0
  69. package/dist/solution/wholebody3d.d.ts +22 -0
  70. package/dist/solution/wholebody3d.d.ts.map +1 -0
  71. package/dist/solution/wholebody3d.js +75 -0
  72. package/dist/types/index.d.ts +52 -0
  73. package/dist/types/index.d.ts.map +1 -0
  74. package/dist/types/index.js +5 -0
  75. package/dist/visualization/draw.d.ts +57 -0
  76. package/dist/visualization/draw.d.ts.map +1 -0
  77. package/dist/visualization/draw.js +400 -0
  78. package/dist/visualization/skeleton/coco133.d.ts +350 -0
  79. package/dist/visualization/skeleton/coco133.d.ts.map +1 -0
  80. package/dist/visualization/skeleton/coco133.js +120 -0
  81. package/dist/visualization/skeleton/coco17.d.ts +180 -0
  82. package/dist/visualization/skeleton/coco17.d.ts.map +1 -0
  83. package/dist/visualization/skeleton/coco17.js +48 -0
  84. package/dist/visualization/skeleton/halpe26.d.ts +278 -0
  85. package/dist/visualization/skeleton/halpe26.d.ts.map +1 -0
  86. package/dist/visualization/skeleton/halpe26.js +70 -0
  87. package/dist/visualization/skeleton/hand21.d.ts +196 -0
  88. package/dist/visualization/skeleton/hand21.d.ts.map +1 -0
  89. package/dist/visualization/skeleton/hand21.js +51 -0
  90. package/dist/visualization/skeleton/index.d.ts +10 -0
  91. package/dist/visualization/skeleton/index.d.ts.map +1 -0
  92. package/dist/visualization/skeleton/index.js +9 -0
  93. package/dist/visualization/skeleton/openpose134.d.ts +357 -0
  94. package/dist/visualization/skeleton/openpose134.d.ts.map +1 -0
  95. package/dist/visualization/skeleton/openpose134.js +116 -0
  96. package/dist/visualization/skeleton/openpose18.d.ts +177 -0
  97. package/dist/visualization/skeleton/openpose18.d.ts.map +1 -0
  98. package/dist/visualization/skeleton/openpose18.js +47 -0
  99. package/docs/ANIMAL_DETECTOR.md +450 -0
  100. package/docs/CUSTOM_DETECTOR.md +568 -0
  101. package/docs/OBJECT_DETECTOR.md +373 -0
  102. package/docs/POSE3D_DETECTOR.md +458 -0
  103. package/docs/POSE_DETECTOR.md +442 -0
  104. package/examples/README.md +119 -0
  105. package/examples/index.html +746 -0
  106. package/package.json +51 -0
  107. package/playground/README.md +114 -0
  108. package/playground/app/favicon.ico +0 -0
  109. package/playground/app/globals.css +17 -0
  110. package/playground/app/layout.tsx +19 -0
  111. package/playground/app/page.tsx +1338 -0
  112. package/playground/eslint.config.mjs +18 -0
  113. package/playground/next.config.ts +34 -0
  114. package/playground/package-lock.json +6723 -0
  115. package/playground/package.json +27 -0
  116. package/playground/postcss.config.mjs +7 -0
  117. package/playground/tsconfig.json +34 -0
  118. package/src/core/base.ts +66 -0
  119. package/src/core/file.ts +141 -0
  120. package/src/core/modelCache.ts +189 -0
  121. package/src/core/posePostprocessing.ts +91 -0
  122. package/src/core/postprocessing.ts +93 -0
  123. package/src/core/preprocessing.ts +127 -0
  124. package/src/index.ts +69 -0
  125. package/src/models/rtmpose.ts +265 -0
  126. package/src/models/rtmpose3d.ts +289 -0
  127. package/src/models/yolo12.ts +220 -0
  128. package/src/models/yolox.ts +214 -0
  129. package/src/solution/animalDetector.ts +955 -0
  130. package/src/solution/body.ts +89 -0
  131. package/src/solution/bodyWithFeet.ts +89 -0
  132. package/src/solution/customDetector.ts +474 -0
  133. package/src/solution/hand.ts +52 -0
  134. package/src/solution/index.ts +10 -0
  135. package/src/solution/objectDetector.ts +816 -0
  136. package/src/solution/pose3dDetector.ts +890 -0
  137. package/src/solution/poseDetector.ts +892 -0
  138. package/src/solution/poseTracker.ts +172 -0
  139. package/src/solution/wholebody.ts +130 -0
  140. package/src/solution/wholebody3d.ts +125 -0
  141. package/src/types/index.ts +62 -0
  142. package/src/visualization/draw.ts +543 -0
  143. package/src/visualization/skeleton/coco133.ts +131 -0
  144. package/src/visualization/skeleton/coco17.ts +49 -0
  145. package/src/visualization/skeleton/halpe26.ts +71 -0
  146. package/src/visualization/skeleton/hand21.ts +52 -0
  147. package/src/visualization/skeleton/index.ts +10 -0
  148. package/src/visualization/skeleton/openpose134.ts +125 -0
  149. package/src/visualization/skeleton/openpose18.ts +48 -0
  150. package/tsconfig.json +32 -0
@@ -0,0 +1,106 @@
1
+ /**
2
+ * PoseTracker - tracks poses across frames with cached detections
3
+ * Reduces detection frequency for better performance
4
+ */
5
+ export class PoseTracker {
6
+ constructor(WholebodyClass, detFrequency = 7, toOpenpose = false, mode = 'balanced', backend = 'onnxruntime', device = 'cpu') {
7
+ this.cachedBoxes = [];
8
+ this.frameCount = 0;
9
+ this.nextId = 0;
10
+ this.detFrequency = detFrequency;
11
+ this.wholebody = new WholebodyClass(null, [640, 640], null, [288, 384], mode, toOpenpose, backend, device);
12
+ }
13
+ async init() {
14
+ await this.wholebody.init();
15
+ }
16
+ async call(image, imgWidth, imgHeight) {
17
+ this.frameCount++;
18
+ // Run detection periodically
19
+ if (this.frameCount % this.detFrequency === 0 || this.cachedBoxes.length === 0) {
20
+ const result = await this.wholebody.call(image, imgWidth, imgHeight);
21
+ this.updateCachedBoxes(result.keypoints, result.scores, imgWidth, imgHeight);
22
+ }
23
+ // Use cached boxes for pose estimation
24
+ const bboxes = this.cachedBoxes.map((tb) => tb.bbox);
25
+ const result = await this.wholebody.call(image, imgWidth, imgHeight, bboxes);
26
+ // Clean up old boxes
27
+ this.cleanupCachedBoxes();
28
+ return result;
29
+ }
30
+ updateCachedBoxes(keypoints, scores, imgWidth, imgHeight) {
31
+ // Simple tracking: create new boxes from keypoints
32
+ const newBoxes = [];
33
+ for (let i = 0; i < keypoints.length; i += 17) {
34
+ const instanceKeypoints = keypoints.slice(i, Math.min(i + 17, keypoints.length));
35
+ const instanceScores = scores.slice(i, Math.min(i + 17, scores.length));
36
+ // Calculate bounding box from keypoints
37
+ let minX = imgWidth;
38
+ let minY = imgHeight;
39
+ let maxX = 0;
40
+ let maxY = 0;
41
+ for (let j = 0; j < instanceKeypoints.length; j++) {
42
+ if (instanceScores[j] > 0.3) {
43
+ const [x, y] = instanceKeypoints[j];
44
+ minX = Math.min(minX, x);
45
+ minY = Math.min(minY, y);
46
+ maxX = Math.max(maxX, x);
47
+ maxY = Math.max(maxY, y);
48
+ }
49
+ }
50
+ if (maxX > minX && maxY > minY) {
51
+ // Add padding
52
+ const padding = 0.25;
53
+ const width = maxX - minX;
54
+ const height = maxY - minY;
55
+ const paddedWidth = width * (1 + padding);
56
+ const paddedHeight = height * (1 + padding);
57
+ const centerX = (minX + maxX) / 2;
58
+ const centerY = (minY + maxY) / 2;
59
+ const x1 = Math.max(0, centerX - paddedWidth / 2);
60
+ const y1 = Math.max(0, centerY - paddedHeight / 2);
61
+ const x2 = Math.min(imgWidth, centerX + paddedWidth / 2);
62
+ const y2 = Math.min(imgHeight, centerY + paddedHeight / 2);
63
+ // Try to match with existing boxes
64
+ let matched = false;
65
+ for (const cachedBox of this.cachedBoxes) {
66
+ const iou = this.calculateIoU([x1, y1, x2, y2], [cachedBox.bbox.x1, cachedBox.bbox.y1, cachedBox.bbox.x2, cachedBox.bbox.y2]);
67
+ if (iou > 0.3) {
68
+ cachedBox.bbox = { x1, y1, x2, y2 };
69
+ cachedBox.age = 0;
70
+ cachedBox.lastSeen = this.frameCount;
71
+ newBoxes.push(cachedBox);
72
+ matched = true;
73
+ break;
74
+ }
75
+ }
76
+ if (!matched) {
77
+ newBoxes.push({
78
+ bbox: { x1, y1, x2, y2 },
79
+ age: 0,
80
+ lastSeen: this.frameCount,
81
+ id: this.nextId++,
82
+ });
83
+ }
84
+ }
85
+ }
86
+ this.cachedBoxes = newBoxes;
87
+ }
88
+ calculateIoU(box1, box2) {
89
+ const x1 = Math.max(box1[0], box2[0]);
90
+ const y1 = Math.max(box1[1], box2[1]);
91
+ const x2 = Math.min(box1[2], box2[2]);
92
+ const y2 = Math.min(box1[3], box2[3]);
93
+ const interWidth = Math.max(0, x2 - x1);
94
+ const interHeight = Math.max(0, y2 - y1);
95
+ const interArea = interWidth * interHeight;
96
+ const box1Area = (box1[2] - box1[0]) * (box1[3] - box1[1]);
97
+ const box2Area = (box2[2] - box2[0]) * (box2[3] - box2[1]);
98
+ const unionArea = box1Area + box2Area - interArea;
99
+ return unionArea > 0 ? interArea / unionArea : 0;
100
+ }
101
+ cleanupCachedBoxes() {
102
+ // Remove boxes that haven't been seen for too long
103
+ const maxAge = this.detFrequency * 3;
104
+ this.cachedBoxes = this.cachedBoxes.filter((box) => this.frameCount - box.lastSeen < maxAge);
105
+ }
106
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Wholebody solution - combines detection and pose estimation
3
+ * Based on rtmlib Wholebody class
4
+ * Supports YOLO12 and YOLOX detectors with RTMW pose model
5
+ */
6
+ import { BBox, ModeType } from '../types';
7
+ export declare class Wholebody {
8
+ private detModel;
9
+ private poseModel;
10
+ private detectorType;
11
+ private static readonly MODE;
12
+ constructor(det?: string | null, detInputSize?: [number, number], pose?: string | null, poseInputSize?: [number, number], mode?: ModeType, toOpenpose?: boolean, backend?: 'onnxruntime', device?: string, detectorType?: 'yolox' | 'yolo12');
13
+ init(): Promise<void>;
14
+ call(image: Uint8Array, imgWidth: number, imgHeight: number, bboxes?: BBox[]): Promise<{
15
+ keypoints: number[][];
16
+ scores: number[];
17
+ }>;
18
+ }
19
+ //# sourceMappingURL=wholebody.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"wholebody.d.ts","sourceRoot":"","sources":["../../src/solution/wholebody.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAe,MAAM,UAAU,CAAC;AAEvD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,YAAY,CAAqB;IAEzC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAmB1B;gBAGA,GAAG,GAAE,MAAM,GAAG,IAAW,EACzB,YAAY,GAAE,CAAC,MAAM,EAAE,MAAM,CAAc,EAC3C,IAAI,GAAE,MAAM,GAAG,IAAW,EAC1B,aAAa,GAAE,CAAC,MAAM,EAAE,MAAM,CAAc,EAC5C,IAAI,GAAE,QAAqB,EAC3B,UAAU,GAAE,OAAe,EAC3B,OAAO,GAAE,aAA6B,EACtC,MAAM,GAAE,MAAc,EACtB,YAAY,GAAE,OAAO,GAAG,QAAkB;IA+CtC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,IAAI,CACR,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,IAAI,EAAE,GACd,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CA0BxD"}
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Wholebody solution - combines detection and pose estimation
3
+ * Based on rtmlib Wholebody class
4
+ * Supports YOLO12 and YOLOX detectors with RTMW pose model
5
+ */
6
+ import { YOLOX } from '../models/yolox';
7
+ import { YOLO12 } from '../models/yolo12';
8
+ import { RTMPose } from '../models/rtmpose';
9
+ export class Wholebody {
10
+ constructor(det = null, detInputSize = [640, 640], pose = null, poseInputSize = [384, 288], mode = 'balanced', toOpenpose = false, backend = 'onnxruntime', device = 'cpu', detectorType = 'yolox') {
11
+ this.detectorType = detectorType;
12
+ // Use mode config if det/pose not specified
13
+ let finalDet = det;
14
+ let finalDetInputSize = detInputSize;
15
+ let finalPose = pose;
16
+ let finalPoseInputSize = poseInputSize;
17
+ if (det === null) {
18
+ finalDet = Wholebody.MODE[mode].det;
19
+ finalDetInputSize = Wholebody.MODE[mode].detInputSize;
20
+ }
21
+ if (pose === null) {
22
+ finalPose = Wholebody.MODE[mode].pose;
23
+ finalPoseInputSize = Wholebody.MODE[mode].poseInputSize;
24
+ }
25
+ // Initialize detector based on type
26
+ if (detectorType === 'yolo12') {
27
+ this.detModel = new YOLO12(finalDet, finalDetInputSize, 0.45, 0.5, backend);
28
+ }
29
+ else {
30
+ this.detModel = new YOLOX(finalDet, finalDetInputSize, 0.45, 0.7, backend);
31
+ }
32
+ this.poseModel = new RTMPose(finalPose, finalPoseInputSize, toOpenpose, backend);
33
+ }
34
+ async init() {
35
+ await this.detModel.init();
36
+ await this.poseModel.init();
37
+ }
38
+ async call(image, imgWidth, imgHeight, bboxes) {
39
+ // Run detection if bboxes not provided
40
+ let finalBboxes = bboxes;
41
+ if (!finalBboxes || finalBboxes.length === 0) {
42
+ const detections = await this.detModel.call(image, imgWidth, imgHeight);
43
+ // Convert Detection[] or BBox[] to BBox[]
44
+ finalBboxes = detections.map((d) => {
45
+ if ('bbox' in d) {
46
+ // Detection type
47
+ return {
48
+ x1: d.bbox.x1,
49
+ y1: d.bbox.y1,
50
+ x2: d.bbox.x2,
51
+ y2: d.bbox.y2,
52
+ };
53
+ }
54
+ // Already BBox type
55
+ return d;
56
+ });
57
+ }
58
+ // Run pose estimation
59
+ const result = await this.poseModel.call(image, imgWidth, imgHeight, finalBboxes);
60
+ return result;
61
+ }
62
+ }
63
+ Wholebody.MODE = {
64
+ performance: {
65
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip',
66
+ detInputSize: [640, 640],
67
+ pose: 'https://download.openmmlab.com/mmpose/v1/projects/rtmw/onnx_sdk/rtmw-dw-x-l_simcc-cocktail14_270e-384x288_20231122.zip',
68
+ poseInputSize: [384, 288],
69
+ },
70
+ lightweight: {
71
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_tiny_8xb8-300e_humanart-6f3252f9.zip',
72
+ detInputSize: [416, 416],
73
+ pose: 'https://download.openmmlab.com/mmpose/v1/projects/rtmw/onnx_sdk/rtmw-dw-l-m_simcc-cocktail14_270e-256x192_20231122.zip',
74
+ poseInputSize: [256, 192],
75
+ },
76
+ balanced: {
77
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip',
78
+ detInputSize: [640, 640],
79
+ pose: 'https://download.openmmlab.com/mmpose/v1/projects/rtmw/onnx_sdk/rtmw-dw-x-l_simcc-cocktail14_270e-256x192_20231122.zip',
80
+ poseInputSize: [256, 192],
81
+ },
82
+ };
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Wholebody3D solution - combines detection and 3D pose estimation
3
+ * Based on rtmlib Wholebody3d class
4
+ * Uses YOLOX detector with RTMW3D pose model
5
+ */
6
+ import { BBox, ModeType, BackendType } from '../types';
7
+ export interface Wholebody3DResult {
8
+ keypoints: number[][][];
9
+ scores: number[][];
10
+ keypointsSimcc: number[][][];
11
+ keypoints2d: number[][][];
12
+ }
13
+ export declare class Wholebody3D {
14
+ private detModel;
15
+ private poseModel;
16
+ private static readonly MODE;
17
+ constructor(det?: string | null, detInputSize?: [number, number], pose?: string | null, poseInputSize?: [number, number], // [width=288, height=384]
18
+ mode?: ModeType, toOpenpose?: boolean, backend?: BackendType);
19
+ init(): Promise<void>;
20
+ call(image: Uint8Array, imgWidth: number, imgHeight: number, bboxes?: BBox[]): Promise<Wholebody3DResult>;
21
+ }
22
+ //# sourceMappingURL=wholebody3d.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"wholebody3d.d.ts","sourceRoot":"","sources":["../../src/solution/wholebody3d.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAe,WAAW,EAAE,MAAM,UAAU,CAAC;AAEpE,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;IACnB,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC7B,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;CAC3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,SAAS,CAAY;IAE7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAmB1B;gBAGA,GAAG,GAAE,MAAM,GAAG,IAAW,EACzB,YAAY,GAAE,CAAC,MAAM,EAAE,MAAM,CAAc,EAC3C,IAAI,GAAE,MAAM,GAAG,IAAW,EAC1B,aAAa,GAAE,CAAC,MAAM,EAAE,MAAM,CAAc,EAAG,0BAA0B;IACzE,IAAI,GAAE,QAAqB,EAC3B,UAAU,GAAE,OAAe,EAC3B,OAAO,GAAE,WAAsB;IAkC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,IAAI,CACR,KAAK,EAAE,UAAU,EACjB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,IAAI,EAAE,GACd,OAAO,CAAC,iBAAiB,CAAC;CA+B9B"}
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Wholebody3D solution - combines detection and 3D pose estimation
3
+ * Based on rtmlib Wholebody3d class
4
+ * Uses YOLOX detector with RTMW3D pose model
5
+ */
6
+ import { YOLOX } from '../models/yolox';
7
+ import { RTMPose3D } from '../models/rtmpose3d';
8
+ export class Wholebody3D {
9
+ constructor(det = null, detInputSize = [640, 640], pose = null, poseInputSize = [288, 384], // [width=288, height=384]
10
+ mode = 'balanced', toOpenpose = false, backend = 'webgpu') {
11
+ // Use mode config if det/pose not specified
12
+ let finalDet = det;
13
+ let finalDetInputSize = detInputSize;
14
+ let finalPose = pose;
15
+ let finalPoseInputSize = poseInputSize;
16
+ if (det === null) {
17
+ finalDet = Wholebody3D.MODE[mode].det;
18
+ finalDetInputSize = Wholebody3D.MODE[mode].detInputSize;
19
+ }
20
+ if (pose === null) {
21
+ finalPose = Wholebody3D.MODE[mode].pose;
22
+ finalPoseInputSize = Wholebody3D.MODE[mode].poseInputSize;
23
+ }
24
+ this.detModel = new YOLOX(finalDet, finalDetInputSize, 0.45, 0.7, backend);
25
+ this.poseModel = new RTMPose3D(finalPose, finalPoseInputSize, toOpenpose, backend);
26
+ }
27
+ async init() {
28
+ await this.detModel.init();
29
+ await this.poseModel.init();
30
+ }
31
+ async call(image, imgWidth, imgHeight, bboxes) {
32
+ // Run detection if bboxes not provided
33
+ let finalBboxes = bboxes;
34
+ if (!finalBboxes || finalBboxes.length === 0) {
35
+ const detections = await this.detModel.call(image, imgWidth, imgHeight);
36
+ // Convert Detection[] or BBox[] to BBox[]
37
+ finalBboxes = detections.map((d) => {
38
+ if ('bbox' in d) {
39
+ // Detection type
40
+ return {
41
+ x1: d.bbox.x1,
42
+ y1: d.bbox.y1,
43
+ x2: d.bbox.x2,
44
+ y2: d.bbox.y2,
45
+ };
46
+ }
47
+ // Already BBox type
48
+ return d;
49
+ });
50
+ }
51
+ // Run 3D pose estimation
52
+ const result = await this.poseModel.call(image, imgWidth, imgHeight, finalBboxes);
53
+ return result;
54
+ }
55
+ }
56
+ Wholebody3D.MODE = {
57
+ performance: {
58
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip',
59
+ detInputSize: [640, 640],
60
+ pose: 'https://huggingface.co/Soykaf/RTMW3D-x/resolve/main/onnx/rtmw3d-x_8xb64_cocktail14-384x288-b0a0eab7_20240626.onnx',
61
+ poseInputSize: [288, 384], // [width=288, height=384] - creates tensor [1,3,384,288]
62
+ },
63
+ lightweight: {
64
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_tiny_8xb8-300e_humanart-6f3252f9.zip',
65
+ detInputSize: [416, 416],
66
+ pose: 'https://huggingface.co/Soykaf/RTMW3D-x/resolve/main/onnx/rtmw3d-x_8xb64_cocktail14-384x288-b0a0eab7_20240626.onnx',
67
+ poseInputSize: [192, 256], // [width=192, height=256]
68
+ },
69
+ balanced: {
70
+ det: 'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip',
71
+ detInputSize: [640, 640],
72
+ pose: 'https://huggingface.co/Soykaf/RTMW3D-x/resolve/main/onnx/rtmw3d-x_8xb64_cocktail14-384x288-b0a0eab7_20240626.onnx',
73
+ poseInputSize: [288, 384], // [width=288, height=384] - creates tensor [1,3,384,288]
74
+ },
75
+ };
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Basic types for rtmlib-ts
3
+ * Based on rtmlib Python library
4
+ */
5
+ export interface Keypoint {
6
+ x: number;
7
+ y: number;
8
+ score: number;
9
+ id: number;
10
+ }
11
+ export interface BodyResult {
12
+ keypoints: Array<Keypoint | null>;
13
+ totalScore: number;
14
+ totalParts: number;
15
+ }
16
+ export type HandResult = Keypoint[];
17
+ export type FaceResult = Keypoint[];
18
+ export interface PoseResult {
19
+ body: BodyResult;
20
+ leftHand: HandResult | null;
21
+ rightHand: HandResult | null;
22
+ face: FaceResult | null;
23
+ }
24
+ export interface BBox {
25
+ x1: number;
26
+ y1: number;
27
+ x2: number;
28
+ y2: number;
29
+ }
30
+ export interface Detection {
31
+ bbox: BBox;
32
+ score: number;
33
+ classId: number;
34
+ }
35
+ export interface ModelConfig {
36
+ det: string;
37
+ detInputSize: [number, number];
38
+ pose: string;
39
+ poseInputSize: [number, number];
40
+ }
41
+ export type ModeType = 'performance' | 'lightweight' | 'balanced';
42
+ export type BackendType = 'opencv' | 'onnxruntime' | 'openvino' | 'wasm' | 'webgpu';
43
+ export type DeviceType = 'cpu' | 'cuda' | 'mps' | string;
44
+ export interface ImageData {
45
+ data: Uint8Array;
46
+ width: number;
47
+ height: number;
48
+ channels: number;
49
+ }
50
+ export type RGBImage = ImageData;
51
+ export type BGRImage = ImageData;
52
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,QAAQ;IACvB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,QAAQ,EAAE,CAAC;AACpC,MAAM,MAAM,UAAU,GAAG,QAAQ,EAAE,CAAC;AAEpC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,UAAU,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,UAAU,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,MAAM,QAAQ,GAAG,aAAa,GAAG,aAAa,GAAG,UAAU,CAAC;AAElE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,aAAa,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;AAEzD,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,SAAS,CAAC;AACjC,MAAM,MAAM,QAAQ,GAAG,SAAS,CAAC"}
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Basic types for rtmlib-ts
3
+ * Based on rtmlib Python library
4
+ */
5
+ export {};
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Drawing utilities for visualization
3
+ * Based on rtmlib Python library
4
+ */
5
+ /**
6
+ * Draw bounding boxes on image
7
+ */
8
+ export declare function drawBbox(img: Uint8Array, width: number, height: number, bboxes: Array<[number, number, number, number]>, color?: [number, number, number]): Uint8Array;
9
+ /**
10
+ * Draw skeleton on image
11
+ */
12
+ export declare function drawSkeleton(img: Uint8Array, width: number, height: number, keypoints: number[][], scores: number[], openposeSkeleton?: boolean, kptThr?: number, radius?: number, lineWidth?: number): Uint8Array;
13
+ /**
14
+ * Draw detections on HTML Canvas
15
+ * @param ctx - Canvas 2D context
16
+ * @param detections - Array of detected objects
17
+ * @param color - Base color for boxes (default: green)
18
+ */
19
+ export declare function drawDetectionsOnCanvas(ctx: CanvasRenderingContext2D, detections: Array<{
20
+ bbox: {
21
+ x1: number;
22
+ y1: number;
23
+ x2: number;
24
+ y2: number;
25
+ confidence: number;
26
+ };
27
+ className?: string;
28
+ }>, color?: string): void;
29
+ /**
30
+ * Draw pose skeleton on HTML Canvas
31
+ * @param ctx - Canvas 2D context
32
+ * @param people - Array of people with keypoints
33
+ * @param confidenceThreshold - Minimum keypoint confidence to draw (default: 0.3)
34
+ */
35
+ export declare function drawPoseOnCanvas(ctx: CanvasRenderingContext2D, people: Array<{
36
+ bbox: {
37
+ x1: number;
38
+ y1: number;
39
+ x2: number;
40
+ y2: number;
41
+ confidence: number;
42
+ };
43
+ keypoints: Array<{
44
+ x: number;
45
+ y: number;
46
+ score: number;
47
+ visible: boolean;
48
+ }>;
49
+ }>, confidenceThreshold?: number): void;
50
+ /**
51
+ * Draw both detections and pose on canvas (convenience method)
52
+ * @param ctx - Canvas 2D context
53
+ * @param results - Detection or pose results
54
+ * @param mode - 'object' or 'pose'
55
+ */
56
+ export declare function drawResultsOnCanvas(ctx: CanvasRenderingContext2D, results: any[], mode?: 'object' | 'pose'): void;
57
+ //# sourceMappingURL=draw.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"draw.d.ts","sourceRoot":"","sources":["../../src/visualization/draw.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAkBH;;GAEG;AACH,wBAAgB,QAAQ,CACtB,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAC/C,KAAK,GAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAe,GAC5C,UAAU,CAgDZ;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EAAE,EAAE,EACrB,MAAM,EAAE,MAAM,EAAE,EAChB,gBAAgB,GAAE,OAAe,EACjC,MAAM,GAAE,MAAY,EACpB,MAAM,GAAE,MAAU,EAClB,SAAS,GAAE,MAAU,GACpB,UAAU,CAoDZ;AAqPD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,wBAAwB,EAC7B,UAAU,EAAE,KAAK,CAAC;IAChB,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC,EACF,KAAK,GAAE,MAAkB,GACxB,IAAI,CAsBN;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,wBAAwB,EAC7B,MAAM,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7E,SAAS,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CAC7E,CAAC,EACF,mBAAmB,GAAE,MAAY,GAChC,IAAI,CAoFN;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,wBAAwB,EAC7B,OAAO,EAAE,GAAG,EAAE,EACd,IAAI,GAAE,QAAQ,GAAG,MAAiB,GACjC,IAAI,CAMN"}