#+title: Integration
#+author: Robert McIntyre
#+email: rlm@mit.edu
#+description:
#+keywords: simulation, jMonkeyEngine3, clojure
#+SETUPFILE: ../../aurellem/org/setup.org
#+INCLUDE: ../../aurellem/org/level-0.org
* Integration
This is the ultimate test which features all of the senses that I've
made so far. The blender file for the creature serves as an example of
a fully equipped creature in terms of senses. You can find it [[../assets/Models/test-creature/hand.blend][here]].
#+begin_html
#+end_html
* Generating the Video
#+name: integration
#+begin_src clojure
(ns cortex.test.integration
"let's play!"
{:author "Robert McIntyre"}
(:use (cortex world util import body sense
hearing touch vision proprioception movement))
(:import (com.jme3.math ColorRGBA Vector3f))
(:import java.io.File)
(:import com.jme3.audio.AudioNode)
(:import com.aurellem.capture.RatchetTimer))
(dorun (cortex.import/mega-import-jme3))
;;(rlm.rlm-commands/help)
(def hand "Models/test-creature/hand.blend")
(def output-base (File. "/home/r/proj/cortex/render/hand"))
#+end_src
For this demonstration I have to manually drive the muscles of the
hand. I do this by creating a little mini-language to describe
simulated muscle contraction.
#+name: integration-2
#+begin_src clojure
(defn motor-control-program
"Create a function which will execute the motor script"
[muscle-positions
script]
(let [current-frame (atom -1)
keyed-script (group-by first script)
current-forces (atom {}) ]
(fn [effectors]
(let [indexed-effectors (vec effectors)]
(dorun
(for [[_ part force] (keyed-script (swap! current-frame inc))]
(swap! current-forces (fn [m] (assoc m part force)))))
(doall (map (fn [effector power]
(effector (int power)))
effectors
(map #(@current-forces % 0) muscle-positions)))))))
(def muscle-positions
[:pointer-2-e
:pointer-2-f
:thumb-1
:thumb-1
:pointer-1-e
:pointer-1-f
:thumb-2-e
:thumb-2-f
:middle-1-e
:middle-1-f
:pointer-3-f
:pointer-3-e
:middle-2-e
:middle-2-f
:middle-3-f
:middle-3-e
:pinky-2-e
:pinky-2-f
:pinky-3-f
:pinky-3-e
:ring-3-e
:ring-3-f
:ring-2-f
:ring-2-e
:ring-1-e
:ring-1-f
:thumb-1-e
:thumb-1-f
:pinky-1-f
:pinky-1-e])
(def full 9001)
;; Choreography:
;; Let the hand fall palm-up
;; it curls its phalanges, starting with the pinky.
;; it lets its phalanges fall back down.
;; block falls down onto the hand, accompanied by a sound. The block
;; can be seen by the hand's eye.
;; hand FORCEFULLY catapults the block so that it hits the camera.
;; the syntax here is [keyframe body-part force]
(def move-fingers
[[300 :pinky-3-f 50]
[320 :pinky-2-f 80]
[340 :pinky-1-f 100]
[310 :ring-3-f 100]
[330 :ring-2-f 120]
[350 :ring-1-f 140]
[330 :middle-3-f 120]
[340 :middle-2-f 120]
[360 :middle-1-f 30]
[350 :pointer-3-f 120]
[360 :pointer-2-f 120]
[380 :pointer-1-f 30]
[800 :pinky-3-f 0]
[800 :pinky-2-f 0]
[800 :pinky-1-f 0]
[800 :ring-3-f 0]
[800 :ring-2-f 0]
[800 :ring-1-f 0]
[800 :middle-3-f 0]
[800 :middle-2-f 0]
[800 :middle-1-f 0]
[800 :pointer-3-f 0]
[800 :pointer-2-f 0]
[800 :pointer-1-f 0]
[800 :pinky-3-e 50]
[800 :pinky-2-e 80]
[800 :pinky-1-e 100]
[800 :ring-3-e 100]
[800 :ring-2-e 120]
[800 :ring-1-e 140]
[800 :middle-3-e 120]
[800 :middle-2-e 120]
[800 :middle-1-e 30]
[800 :pointer-3-e 120]
[800 :pointer-2-e 120]
[800 :pointer-1-e 30]
[870 :pinky-3-e 0]
[870 :pinky-2-e 0]
[870 :pinky-1-e 0]
[870 :ring-3-e 0]
[870 :ring-2-e 0]
[870 :ring-1-e 0]
[870 :middle-3-e 0]
[870 :middle-2-e 0]
[870 :middle-1-e 0]
[870 :pointer-3-e 0]
[870 :pointer-2-e 0]
[870 :pointer-1-e 0]
[1500 :pointer-1-f full]
[1500 :pointer-2-f full]
[1500 :pointer-3-f full]
[1500 :middle-1-f full]
[1500 :middle-2-f full]
[1500 :middle-3-f full]
[1510 :pointer-1-f 0]
[1510 :pointer-2-f 0]
[1510 :pointer-3-f 0]
[1510 :middle-1-f 0]
[1510 :middle-2-f 0]
[1510 :middle-3-f 0]
])
(defn gen-summon-ball [debug?]
(let [wait (atom 1100)]
(fn [world]
(if (= 0 (swap! wait dec))
(let [brick
(box 0.8 0.8 0.8 :mass 0.05
:position (Vector3f. -0.5 0 0.5)
:color (ColorRGBA/Red))
bell (AudioNode. (asset-manager)
"Sounds/pure.wav" false)]
(.play bell)
(if debug?
(.addControl
brick
(proxy [AbstractControl] []
(controlUpdate [tpf]
(println-repl (.getWorldTranslation brick)))
(controlRender [_ _]))))
(add-element world brick))))))
(import com.aurellem.capture.Capture)
(defn test-integration
"Testing Everything!
You will see an articulated hand fall onto the table. It has all
senses including:
- Vision, 4 channels
- Hearing
- Touch
- Proprioceptoin
- Muscle Tension
Keys:
: fire ball"
([] (test-integration false))
([record?]
(let [me (sphere 0.5 :color ColorRGBA/Blue :physical? false)
base (File. "/home/r/proj/cortex/render/hand")
creature (doto (load-blender-model hand)
(body!))
summon-ball (gen-summon-ball false)
;;;;;;;;;;;; Sensors/Effectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
touch (touch! creature)
touch-display (view-touch)
vision (vision! creature)
vision-display (view-vision)
hearing (hearing! creature)
hearing-display (view-hearing)
prop (proprioception! creature)
prop-display (view-proprioception)
control-script (motor-control-program
muscle-positions move-fingers)
muscles (movement! creature)
muscle-display (view-movement)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
fix-display (gen-fix-display)]
(world
(nodify [creature
(box 10 2 10 :position (Vector3f. 0 -9 0)
:color ColorRGBA/Gray :mass 0)
me])
standard-debug-controls
(fn [world]
(.setTimer world (RatchetTimer. 60))
(position-camera
world (Vector3f. -0.13217318, 5.816415, -5.3089414)
(Quaternion. 0.55685693, 0.0042774677, -0.0028673497, 0.83059245))
(light-up-everything world)
(enable-debug world)
(add-camera! world
(add-eye! creature
(.getChild
(.getChild creature "eyes") "eye"))
(comp (view-image) BufferedImage!))
(if record?
(Capture/captureVideo
world (File. base "main")))
(if record?
(Capture/captureAudio
world (File. base "main.wav"))))
(fn [world tpf]
(prop-display
(prop)
(if record? (File. base "proprio")))
(touch-display
(map #(% (.getRootNode world)) touch)
(if record? (File. base "touch")))
(vision-display
(map #(% world) vision)
(if record? (File. base "vision")))
(hearing-display
(map #(% world) hearing)
(if record? (File. base "hearing")))
(muscle-display
(control-script muscles)
(if record? (File. base "muscle")))
(summon-ball world)
(.setLocalTranslation me (.getLocation (.getCamera world)))
(fix-display world))))))
#+end_src
** ImageMagick / ffmpeg
Just a bunch of calls to imagemagick to arrange the data that
=test-everything!= produces.
#+name: magick-8
#+begin_src clojure
(ns cortex.video.magick8
(:import java.io.File)
(:use clojure.java.shell))
(comment
;; list of touch targets
0 middle-11
1 middle-21
2 middle-31
3 pinky-11
4 pinky-21
5 pinky-31
6 pointer-11
7 pointer-21
8 pointer-31
9 ring-11
10 ring-21
11 ring-31
12 thumb-11
13 thumb-2.0011
;; list of vision targets
0 :all
1 :green
2 :blue
3 :red
;; list of proprio targets
0 middle-11 -> middle-21
1 middle-21 -> middle-31
2 thumb-11 -> thumb-2.0011
3 pointer-11 -> pointer-21
4 pointer-21 -> pointer-31
5 ring-21 -> ring-31
6 ring-11 -> ring-21
7 pinky-21 -> pinky-31
8 pinky-11 -> pinky-21
9 middle-11 -> palm1
10 pinky-11 -> palm1
11 palm1 -> pointer-11
12 palm1 -> ring-11
13 palm1 -> thumb-11
;; list of muscle targets
0 :pointer-2-e
1 :pointer-2-f
2 :thumb-1
3 :thumb-1
4 :pointer-1-e
5 :pointer-1-f
6 :thumb-2-e
7 :thumb-2-f
8 :middle-1-e
9 :middle-1-f
10 :pointer-3-f
11 :pointer-3-e
12 :middle-2-e
13 :middle-2-f
14 :middle-3-f
15 :middle-3-e
16 :pinky-2-e
17 :pinky-2-f
18 :pinky-3-f
19 :pinky-3-e
20 :ring-3-e
21 :ring-3-f
22 :ring-2-f
23 :ring-2-e
24 :ring-1-e
25 :ring-1-f
26 :thumb-1-e
27 :thumb-1-f
28 :pinky-1-f
29 :pinky-1-e
)
(def base (File. "/home/r/proj/cortex/render/hand"))
(defn prepare-muscle [muscle]
["(" muscle "-rotate" "90" "-scale" "15x60!" ")"])
(defn prepare-touch [touch]
["(" touch "-rotate" "180" ")"])
(defn generate-top-finger [tip-flexor tip-extensor tip
joint-2-3
mid-flexor mid-extensor mid
joint-1-2]
["("
"-size" "113x357" "xc:transparent"
(prepare-muscle tip-flexor) "-geometry" "+0+7" "-composite"
(prepare-muscle tip-extensor) "-geometry" "+98+7" "-composite"
(prepare-touch tip) "-geometry" "+18+0" "-composite"
joint-2-3 "-geometry" "+32+79" "-composite"
(prepare-muscle mid-flexor) "-geometry" "+19+131" "-composite"
(prepare-muscle mid-extensor) "-geometry" "+80+131" "-composite"
(prepare-touch mid) "-geometry" "+39+133" "-composite"
joint-1-2 "-geometry" "+32+193" "-composite"
")"])
(defn file-names [#^File dir]
(map #(.getCanonicalPath %) (next (sort (file-seq dir)))))
(defn file-groups [& paths]
(apply (partial map list )
(map (comp file-names #(File. base %))
paths)))
(defn pinky []
(file-groups
"muscle/18"
"muscle/19"
"touch/5"
"proprio/7"
"muscle/17"
"muscle/16"
"touch/4"
"proprio/8"
"muscle/28"
"muscle/29"
"touch/3"
"proprio/10"))
(defn ring []
(file-groups
"muscle/21"
"muscle/20"
"touch/11"
"proprio/5"
"muscle/22"
"muscle/23"
"touch/10"
"proprio/6"
"muscle/25"
"muscle/24"
"touch/9"
"proprio/12"))
(defn middle []
(file-groups
"muscle/14"
"muscle/15"
"touch/2"
"proprio/1"
"muscle/13"
"muscle/12"
"touch/1"
"proprio/0"
"muscle/9"
"muscle/8"
"touch/0"
"proprio/9"))
(defn pointer []
(file-groups
"muscle/10"
"muscle/11"
"touch/8"
"proprio/4"
"muscle/1"
"muscle/0"
"touch/7"
"proprio/3"
"muscle/5"
"muscle/4"
"touch/6"
"proprio/11"))
(defn thumb []
(file-groups
"muscle/7"
"muscle/6"
"touch/13"
"proprio/2"
"muscle/27"
"muscle/26"
"muscle/3"
"muscle/2"
"touch/12"
"proprio/13"))
(defn generate-finger
[tip-flexor tip-extensor tip
joint-2-3
mid-flexor mid-extensor mid
joint-1-2
base-flexor base-extensor base
joint-palm-1]
["("
"-size" "113x357" "xc:transparent"
(generate-top-finger
tip-flexor tip-extensor tip
joint-2-3
mid-flexor mid-extensor mid
joint-1-2) "-geometry" "+0+0" "-composite"
(prepare-muscle base-flexor) "-geometry" "+19+245" "-composite"
(prepare-muscle base-extensor) "-geometry" "+80+245" "-composite"
(prepare-touch base) "-geometry" "+39+247" "-composite"
joint-palm-1 "-geometry" "+32+307" "-composite"
")"])
(defn generate-thumb
[tip-flexor tip-extensor tip
joint-1-2
mid-flexor mid-extensor mid-flexor-2 mid-extensor-2 mid
joint-palm-1]
["("
"-size" "113x357" "xc:transparent"
(generate-top-finger
tip-flexor tip-extensor tip
joint-1-2
mid-flexor mid-extensor mid
joint-palm-1) "-geometry" "+0+0" "-composite"
(prepare-muscle mid-flexor-2) "-geometry" "+2+131" "-composite"
(prepare-muscle mid-extensor-2) "-geometry" "+100+131" "-composite"
")"])
(defn generate-hand
[pinky-pieces
ring-pieces
middle-pieces
pointer-pieces
thumb-pieces]
["("
"-size" "688x769" "xc:transparent"
(apply generate-finger pinky-pieces)
"-geometry" "+0+195" "-composite"
(apply generate-finger ring-pieces)
"-geometry" "+111+100" "-composite"
(apply generate-finger middle-pieces)
"-geometry" "+228+0" "-composite"
"(" (apply generate-thumb thumb-pieces) "-background" "#00000000"
"-rotate" "45" ")"
"-geometry" "+300+420" "-composite"
(apply generate-finger pointer-pieces)
"-geometry" "+350+96" "-composite"
")"])
(defn generate-vision
[all green blue red]
["("
"-size" "204x192" "xc:transparent"
all "-geometry" "+0+0" "-composite"
green "-geometry" "+113+0" "-composite"
blue "-geometry" "+0+105" "-composite"
red "-geometry" "+113+105" "-composite"
")"])
(def test-muscle (File. base "muscle/0/0000000.png"))
(def test-proprio (File. base "proprio/0/0000000.png"))
(def test-tip (File. base "touch/2/0000000.png"))
(def test-mid (File. base "touch/0/0000000.png"))
(def test-vision (File. base "vision/0/0000000.png"))
(def test-hearing (File. base "hearing/0/0000000.png"))
(def test-main (File. base "main/0000000.png"))
(def test-target (File. base "output.png"))
(def background (File. base "background.png"))
(use 'clojure.java.shell)
(defn vision []
(file-groups
"vision/0"
"vision/1"
"vision/2"
"vision/3"))
(defn hearing []
(file-names (File. base "hearing/0")))
(defn main []
(file-names (File. base "main")))
(defn targets [dest max]
(map
(comp #(.getCanonicalPath %)
#(File. (str base dest (format "%07d.png" %))))
(range max)))
(defn final-image [main [all red green blue] hearing
pinky ring middle pointer thumb target]
(println target)
(apply
sh
(flatten
["convert"
(.getCanonicalPath background)
(generate-hand pinky ring middle pointer thumb)
"-geometry" "+809+22" "-composite"
(generate-vision all red green blue)
"-geometry" "+974+599" "-composite"
hearing
"-geometry" "+784+819" "-composite"
main
"-geometry" "+78+202" "-composite"
target])))
(defn combine-files []
(dorun
(pmap final-image
(main)
(vision)
(hearing)
(pinky)
(ring)
(middle)
(pointer)
(thumb)
(targets "/out/" (count (main))))))
(defn subtitles []
(file-names (File. base "subs")))
(defn outs []
(file-names (File. base "out")))
(defn mix-subtitles []
(let [subs (subtitles)
targets (targets "/out-subs/" (count subs))
overlay (.getCanonicalPath (File. base "output.png"))]
(dorun
(pmap
(fn [sub target]
(sh
"convert"
overlay
sub "-geometry" "+0+0" "-composite"
target))
subs targets))))
(defn out-subtitles []
(file-names (File. base "out-subs")))
(defn insert-subtitles []
(let [subtitles (out-subtitles)
outs (outs)
targets (targets
"/final/"
(+ (count outs) (count subtitles)))]
(dorun
(pmap
#(sh "cp" %1 %2)
(concat subtitles outs) targets))))
(defn generate-final []
(combine-files)
(mix-subtitles)
(insert-subtitles))
#+end_src
#+begin_src sh :results silent
cd /home/r/proj/cortex/render/hand
sox --ignore-length main.wav main-delayed.wav delay 24
mogrify -resize 755x final/*
#+end_src
#+begin_src sh :results silent
cd /home/r/proj/cortex/render/hand
ffmpeg -r 60 -i final/%07d.png -i main-delayed.wav -b:a 128k \
-b:v 9000k -c:a libvorbis -c:v libtheora hand.ogg
#+end_src
* Source Listing
- [[../src/cortex/test/integration.clj][cortex.test.integration]]
- [[../src/cortex/video/magick8.clj][cortex.video.magick8]]
- [[../assets/Models/subtitles/hand.blend][subtitles/hand.blend]]
#+html:
- [[http://hg.bortreb.com ][source-repository]]
* COMMENT purgatory
#+begin_src clojure
(defn bullet-trans* []
(let [obj-a (box 1.5 0.5 0.5 :color ColorRGBA/Red
:position (Vector3f. 5 0 0)
:mass 90)
obj-b (sphere 0.5 :color ColorRGBA/Blue
:position (Vector3f. -5 0 0)
:mass 0)
control-a (.getControl obj-a RigidBodyControl)
control-b (.getControl obj-b RigidBodyControl)
move-up? (atom nil)
move-down? (atom nil)
move-left? (atom nil)
move-right? (atom nil)
roll-left? (atom nil)
roll-right? (atom nil)
force 100
swivel
(.toRotationMatrix
(doto (Quaternion.)
(.fromAngleAxis (/ Math/PI 2)
Vector3f/UNIT_X)))
x-move
(doto (Matrix3f.)
(.fromStartEndVectors Vector3f/UNIT_X
(.normalize (Vector3f. 1 1 0))))
timer (atom 0)]
(doto
(ConeJoint.
control-a control-b
(Vector3f. -8 0 0)
(Vector3f. 2 0 0)
;;swivel swivel
;;Matrix3f/IDENTITY Matrix3f/IDENTITY
x-move Matrix3f/IDENTITY
)
(.setCollisionBetweenLinkedBodys false)
(.setLimit (* 1 (/ Math/PI 4)) ;; twist
(* 1 (/ Math/PI 4)) ;; swing span in X-Y plane
(* 0 (/ Math/PI 4)))) ;; swing span in Y-Z plane
(world (nodify
[obj-a obj-b])
(merge standard-debug-controls
{"key-r" (fn [_ pressed?] (reset! move-up? pressed?))
"key-t" (fn [_ pressed?] (reset! move-down? pressed?))
"key-f" (fn [_ pressed?] (reset! move-left? pressed?))
"key-g" (fn [_ pressed?] (reset! move-right? pressed?))
"key-v" (fn [_ pressed?] (reset! roll-left? pressed?))
"key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})
(fn [world]
(enable-debug world)
(set-gravity world Vector3f/ZERO)
)
(fn [world _]
(if @move-up?
(.applyForce control-a
(Vector3f. force 0 0)
(Vector3f. 0 0 0)))
(if @move-down?
(.applyForce control-a
(Vector3f. (- force) 0 0)
(Vector3f. 0 0 0)))
(if @move-left?
(.applyForce control-a
(Vector3f. 0 force 0)
(Vector3f. 0 0 0)))
(if @move-right?
(.applyForce control-a
(Vector3f. 0 (- force) 0)
(Vector3f. 0 0 0)))
(if @roll-left?
(.applyForce control-a
(Vector3f. 0 0 force)
(Vector3f. 0 0 0)))
(if @roll-right?
(.applyForce control-a
(Vector3f. 0 0 (- force))
(Vector3f. 0 0 0)))
(if (zero? (rem (swap! timer inc) 100))
(.attachChild
(.getRootNode world)
(sphere 0.05 :color ColorRGBA/Yellow
:physical? false :position
(.getWorldTranslation obj-a)))))
)
))
(defn test-joint [joint]
(let [[origin top bottom floor] (world-setup joint)
control (.getControl top RigidBodyControl)
move-up? (atom false)
move-down? (atom false)
move-left? (atom false)
move-right? (atom false)
roll-left? (atom false)
roll-right? (atom false)
timer (atom 0)]
(world
(nodify [top bottom floor origin])
(merge standard-debug-controls
{"key-r" (fn [_ pressed?] (reset! move-up? pressed?))
"key-t" (fn [_ pressed?] (reset! move-down? pressed?))
"key-f" (fn [_ pressed?] (reset! move-left? pressed?))
"key-g" (fn [_ pressed?] (reset! move-right? pressed?))
"key-v" (fn [_ pressed?] (reset! roll-left? pressed?))
"key-b" (fn [_ pressed?] (reset! roll-right? pressed?))})
(fn [world]
(light-up-everything world)
(enable-debug world)
(set-gravity world (Vector3f. 0 0 0))
)
(fn [world _]
(if (zero? (rem (swap! timer inc) 100))
(do
;; (println-repl @timer)
(.attachChild (.getRootNode world)
(sphere 0.05 :color ColorRGBA/Yellow
:position (.getWorldTranslation top)
:physical? false))
(.attachChild (.getRootNode world)
(sphere 0.05 :color ColorRGBA/LightGray
:position (.getWorldTranslation bottom)
:physical? false))))
(if @move-up?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. 0 0 10))))
(if @move-down?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. 0 0 -10))))
(if @move-left?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. 0 10 0))))
(if @move-right?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. 0 -10 0))))
(if @roll-left?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. -1 0 0))))
(if @roll-right?
(.applyTorque control
(.mult (.getPhysicsRotation control)
(Vector3f. 1 0 0))))))))
#+end_src
* COMMENT generate source
#+begin_src clojure :tangle ../src/cortex/test/integration.clj
<>
<>
#+end_src
#+begin_src clojure :tangle ../src/cortex/video/magick8.clj
<>
#+end_src