DH2642 Interaction
INTERACTION,* EVENTS* Cris%an(Bogdan( cris%@kth.se( (
DH2642 Interaction Programming
“Kinds”(of(interac%on(
• User(interacts(with(a(widget((component),(producing(an( event,(this(lecture( • Applica%on(signaling(an(event((%mer,(sensor(value(change,( new(data(from(the(net)(to(the(user((no%fica%ons,(men%oned( later)( • Mul%touch(and(Gestures((men%oned(in(this(lecture)(( • Drag(and(Drop((and(copy(paste)(within(an(applica%on(or( between(applica%ons((coming(lectures)( • Automa%c(appearance(changes(on(window(size/font/skin( change((previous(lecture,(events(this(lecture)( • Interac%on(Design(paLerns((men%oned(today(for(Android)(
DH2642 Interaction Programming DH2642 Interaction
Interac%on(process(
• User(interacts((press(a(buLon(widget)( • The(computer(detects(a(change(in(its(input(ports(( (mouse(click)( • Events(are(generated((((MouseEvent(and(Ac%onEvent)( • You(as(a(programmer(decided(that(you(are(interested( in((subscribe(to)(such(events((presses(on(that(buLon)( • When(subscribing,(you(registered(some(piece(of(code( (event(handler)(to(run(when(such(events(come( • Your(handler(is(called(and(treats(the(event(
DH2642 Interaction Programming
A(buLon(event((Q(HTML( ( Layout(and(event(linking:(
DH2642 Interaction Programming DH2642 Interaction
A(buLon(event(Q(Android(
Layout:'' '
A(buLon(event(–(Java(Swing(
Layout/ini%aliza%on:' JButton'b'='new'JButton("Press'me");'' …' b.addActionListener(new'ButtonPressDetector());' ' ' Listener:' public'class'ButtonPressDetector'' 'implements'ActionListener{' 'public'void'actionPerformed(ActionEvent'e){' ''System.out.println("button'was'pressed!'"'+'e);' '}' }' ' DH2642 Interaction Programming
A(buLon(event((Q(HTML(–(beLer(way( ( Layout(and(event(linking:(
A(buLon(event(–(FXML(Javascript(
33 '…' ' DH2642 Interaction Programming Handlers(for(mul%ple(events( • Different(event(types( – In(Java/Android,(a(class(can(implement(many( listeners( (implements(Ac%onListener,(MouseListener(( – Since(JavaScript(is(weakly(typed,(this(is(no(issue( • One(event(type,(from(many(sources:( ButtonPressDetector'detector='' ' 'new'ButtonPressDetector();( ''b1.addActionListener(detector);' ''b2.addActionListener(detector);' ( DH2642 Interaction Programming DH2642 Interaction Events(at(different(levels( • a(buLon(press(can(originate(from(one(or(more(lowQlevel( events( – HTML(onClick,(or(onKeyDown(followed(by(onKeyUp(Q>( keyPress( – Android:(Mo%onEvent(Q>(View.OnClickListener( – Swing:(MouseEvent,(KeyEvent((( • MouseEvents:(mouse(pressed(+(mouse(released=(mouse(clicked( • key(typed((press(Tab(to(bring(the(buLon(in(focus),(key(typed((press( Space(or(Enter)( ( DH2642 Interaction Programming ?( What(are(possible(the(lowQlevel(events(that( generate(the(following(highQlevel(events:( – moving(the(text(cursor(of(a(textbox(( – moving(the(keyboard(focus(from(one(textbox(to(another( scrolling(a(textbox(( – closing(a(window( DH2642 Interaction Programming DH2642 Interaction Answers( – moving(the(text(cursor(of(a(textbox( • Mouse(click(in(the(textbox( • Arrow(keys(( • Ctrl+A,(Ctrl+E,(etc( – moving(the(keyboard(focus(from(one(textbox(to(another( • Click(in(the(textbox( • Tab,(or(shiaQTab(un%l(the(focus(gets(there( – scrolling(a(textbox( • Mouse(clicks(on(the(scrollbar( • Arrow(keys,(page(up,(page(down,(Ctrl+…( – closing(a(window( • Mouse(click(on(the(close(handle( • CtrlQW,(CommandQW(etc( DH2642 Interaction Programming Low*level* Key* onkeydown,3onkeyup,3onkeypress,3 3 .keydown();.keyup();3.keypress();33 javax.awt.event.KeyListener3 keyPressed(),3keyReleased(),3keyTyped()3 3 javafx.scene.input.KeyEvent3 android.view.View.OnKeyListener3 3 onKey()3 DH2642 Interaction Programming DH2642 Interaction ?( My(handler(method(is(called(so(I(know(that(a(key(is(pressed.( How(to(figure(out(which(key(was(pressed?( e.g.(Swing:' public'class'KeyDetector'implements'KeyListener{' 'public'void'keyPressed(KeyEvent'ke){' ' ''?????' '}' …' }' e.g.(JavaFX:( public'class'KeyDetector'implements'EventHandler Answer( This(is(the(role(of(the(event(object((ke)( All(eventQspecific(details(are(in(the(event(object( passed(to(the(event(handler( • HTML:(event.keyCode( • JavaFX:(ke.getCode()( • Swing:(ke.getKeyChar()( ( DH2642 Interaction Programming DH2642 Interaction The(event(object( • Gives(details(about(what(exactly(happened,(and(the( context,(for(example( – which(widget(generated(the(event( – %mestamp(of(the(event( – mouse(posi%on(during(mouse(clicks(or(mouse(drags( – mouse(buLon(status( – modifier(key((Alt,(Ctrl)(status(during(mouse(events(and(key( events( – and(many(more,(depending(on(the(event(type( DH2642 Interaction Programming Low*level* Mouse*/*Touch* onclick,3ondblclick,3onmousedown,3onmouseup,3 onmouseover,3onmouseout,3onmousemove3 3 .click(),3.dblclick(),3.mousedown(),33 .mouseup(),3.mouseover(),3.mouseout()3…3 javax.awt.event.MouseListener3 mouseClicked()3=3mousePressed()3+3mouseReleased()3 mouseEntered(),3mouseExited()33 3 javafx.scene.input.MouseEvent3 android.view.View.OnClickListener333333onClick()3 android.view.View.OnLongClickListener33onLongClick()3 android.view.View.OnTouchListener333333onTouch()3 ' DH2642 Interaction Programming DH2642 Interaction Low*level* (Mouse)*Mo=on* ondrag,3ondragend,3ondragenter,3 ondragleave,3ondragover,3ondragstart,3 ondrop3 javax.awt.event.MouseMotionListener3 mouseDragged(),3mouseMoved()3 3 javafx.scene.input.MouseDragEvent3 android.view.View.OnDragListener3 3 onDragEvent()3 DH2642 Interaction Programming ?( • My(mouse(mo%on(handler(will(be(called(for( every(liLle(mouse(move.(Won’t(the(%me( needed(for(my(mouse(mo%on(handler’s( execu%on(make(my(applica%on(unresponsive?( DH2642 Interaction Programming DH2642 Interaction Answer( • Yes(your(event(handler(code(can(block(the(event(dispatcher(if( it(e.g.(waits(forever(or(does(%meQconsuming(opera%ons( • That(will(affect(both(your(code(and(all(other(event(handlers( ran(on(the(respec%ve(plaiorm( • If(you(want(your(applica%on(to(feel(responsive,(op%mize(the( %me(needed(for(your(event(handling(code( • If(complex(opera%ons(are(needed,(start(them(in(another( thread( – Or(use(asynchronous(callse.g.(in(Javascript( – That(is,(specify(a(callback(to(be(called(when(the(%meQconsuming( opera%on(ends( DH2642 Interaction Programming ?( • Do(events(always(come(from(a(user(ac%on?( DH2642 Interaction Programming DH2642 Interaction Answer(( • No.(( • Example:(a(temperature(sensor(can(trigger(an(event( if(the(temperature(changes(over(a(certain(threshold( – In(order(to(avoid(too(many(events(being(generated,( sensors(are(typically(configured(before(being(subscribed(to( (thresholds,(accuracy,(etc)( • Events(can(be(triggered(in(soaware.(Key(and(Mouse( events(can(be(built( • Also(higher(level(events(can(be(triggered(by(e.g.( hiding(and(showing(widgets(or(windows( DH2642 Interaction Programming ?( • Are(listener(methods(always(triggered(by( hardware?( ( DH2642 Interaction Programming DH2642 Interaction Answer( No,(any(change(in(hardware(or(soaware(can(trigger(listener( methods( • java.beans.PropertyChangeListener3 • javafx.beans.value.ChangeListener Event(architecture( • The(system(places( changes(in(a(queue( Interrupt-callback Input device • The(framework( consumes(events(from( Callback- the(queue(regularly( list • Send(an(event(instance( Event queue with((event(details(to(all( Callback interested(((subscribed)( par%es((dispatch)( Dispatcher • Each(subscribed(party( will(register(a(callback( DH2642 Interaction Programming DH2642 Interaction Higher*level* (Keyboard)*Focus* onfocus,3onblur3 3 .focus(),3.blur(),33 .focusin(),3.focusout()3 javax.awt.event.FocusListener3 focusGained();3focusLost()3 3 JavaFX:(add(a(ChangeListener(to(the(widget’s(focusedProperty3 android.view.View.FocusChangeListener3 3 onFocusChange()3 DH2642 Interaction Programming High*level* (Text)*Change* onchange3 3 .change()3 javax.awt.event.TextListener3 textValueChanged()3 3 3 JavaFX: add a ChangeListener to the widget’s textProperty3 android.text.TextWatcher3 3 onTextChanged();3afterTextChanged();3 beforeTextChangedDH2642 Interaction Programming();3 DH2642 Interaction High*level* HTML*specific* onselect3 Selec%on(of(text(in(an(element( .select()3 3 onresize3 .resize()3 Size(of(element(changes( 3 onload3 Object(has(loaded.(Can(be(used(for(the(whole(page,( .load()3 but(beLer(use(jQuery(.ready()( 3 onscroll3 .scroll()3 Scrolling(to(a(different(place(in(the(element( 3 onunload3 .unload()3 Before(the(browser(closes(the(document( DH2642 Interaction Programming High*level* Java*Swing*specific* javax.awt.event.ActionListener3 3 3actionPerformed()3 javax.awt.event.AdjustmentListener 3adjustmentValueChanged()3 javax.awt.event.ItemListener 3 3itemStateChanged()3 ' ' javax.awt.event.WindowListener 3 3windowActivated()3 333333windowClosed()3 333333windowOpened()3 333333…3 ' ' javax.awt.event.ComponentListener 3componentHidden()3 333333componentMoved()3 333333componentResized()3 333333componentShown()3 javax.awt.event.ContainerListener 3componentAdded()3 333333componentRemoved()3 3 and3many,3many,3many3more…3DH2642 Interaction Programming ' DH2642 Interaction High*level* Android*specific* android.hardware.SensorListener33 3onAccuracyChanged()3 333333onSensorChanged()3 3 android.view.GestureDetector.OnGestureListener3 3onDown()3 333333onFling()3 333333onScroll()3 333333…3 3 android.view.View.OnCreateContextMenuListener3 3onCreteContextMenu()3 3 3 3 3 3 and3many,3many,3many3more…3 DH2642 Interaction Programming ?( b.addActionListener(new'ButtonPressDetector());' …' class'ButtonPressDetector'implements'EventHandler DH2642 Interaction Programming DH2642 Interaction Answers( • Javascript(handlers(are(typically(faster(to(write( – but(there(are(shorter(ways(to(write(the(same(code(in(Java( • Java(code(is(safer(( – the(type(of(the(event(and(the(event(proper%es(accessed(can(be( checked(at(compile(%me( • event.mousePosi=on(is(typically(a(wrong(property(access(for(a(key(event( but(it(will(only(be(found(at(run%me(in(Javascript( • Some%mes(the(illegal(access(only(occurs(aPer*a*few*years*in*produc=on…( • Java(code(is(faster(to(execute(as(it(is(preQcompiled(and(not( interpreted( – On(the(other(hand,(interpreta%on(gives(the(programmer(many(other( possibili%es( – That’s(why(you(can(call(Java(from(Javascript(but(the(reverse(is(not(easy( DH2642 Interaction Programming A(buLon(event(–(JavaFX((recap)( Layout:'' '' A(buLon(event(–(JavaFX((shorter)( Layout:'' '' DH2642 Interaction Programming A(buLon(event(–(JavaFX((shortest)( Layout:'' '' The compiler figures types out as follows: setOnAction() called => EventHandler Lambda expression, or closure (Java 8) works for any single-method interface (e.g. Runnable) DH2642 Interaction Programming DH2642 Interaction Event(consump%on( • LowQlevel(events(can(be(consumed(( • So(they(will(not(lead(to(higherQlevel(events,(and(don’t(bubble( • Example:(avoiding(text(being(typed(in(a(textbox( • HTML/JS:(event.stopPropaga%on()( • Swing,(JavaFX:(event.consume()( • Android(Long(click,(Key(and(Touch(event(can(be(consumed)by( returning(true(in(their(handler(methods( ' public'class'KeyConsumer333implements'''View.KeykListener{' 'public'boolean3 'onKeyDown''(View'v,'Editable'text,'int'keyCode,'KeyEvent' event'){'' '''''''...'return'true;'}' //'…'other'KeyListener'methods' DH2642 Interaction Programming }' Event(capturing(and(bubbling( 1. Capturing((down(the(tree)( – the(event(moves(from(the(widget(tree(root(to(the(lowestQrank(relevant( widget(s)( – any(of(the(parent(components(can(filter(out(or(modify(the(event( 2. event(listeners(associated(to(the(lowestQrank(widgets(are( called( 3. Bubbling((up(the(tree)( – the(event(bubbles(up(to(the(parent(widgets,(so*their*event*listeners* are*called* • To(stop(events(from(bubbling,(consume(them!( • Not(all(events(bubble!(Check(your(plaiorm!( – Many(HTML(events(bubble((mouse,(key,(but(not(focus,(submit)( – Many(lowQlevel(JavaFX(events(bubble( DH2642 Interaction Programming DH2642 Interaction Event(capturing( • JavaFX(event(filters( – Take(effect(during(event(capturing( – Used(on(en%re(subtrees(to( • Handle(all(subtypes(of(an(event(in(common( • Ignore(events((by(consuming(them)(( • Javascript,(pass(true(to(addEventListener( – node.addEventListener(eventType,(handler,(true)( – the(listener(will(then(be(called*during*capturing,( not(during(bubbling( • and(can(call(event.stopPropaga%on()( DH2642 Interaction Programming ?( • Can(one(event(be(relevant((captured)(for(two( sibling(widgets?( Frame Menu Panel Sibling1 … Sibling2 … … DH2642 Interaction Programming DH2642 Interaction Answer( • In(most(cases(no,(as(the(layout(manager(will(not( place(them(on(top(of(each(other( • However,(in(special(cases(sibling(widgets(may( superimpose* – Even(then,(the(“rightmost”(widget((Sibling2)(is(considered( on(top(of(Sibling1( – so(events(on(Sibling2(may(go(to(Sibling1(but(not(the(other( way(around( • In(JavaFX(Shapes(are(widgets.(( – You(can(have(a(triangle(shape(in(a(tree.( – NonQrectangular(shapes(can(easily(superimpose( DH2642 Interaction Programming Android(Mul%touch(interac%on( http://www.zdnet.com/blog/burnette/how-to-use-multi-touch-in- android-2/1747 • (a)(Tap,((b)(drag,((c)(pinch(zoom( • Mul%touch(is(detected(via(the(View.OnTouchListener( (public(boolean(onTouch(View(v,(Mo%onEvent(event)(( • Mo%onEvent.getAc%on()(type(of(ac%on( – ACTION_DOWN(first(finger(down( – ACTION_POINTER_DOWN(2nd(,(3rd((finger(down( – ACTION_MOVE((move(any(numbe(of(fingers( – ACTION_POINTER_UP(one(finguer(up( – ACTION_UP(last(finger(up( • Mo%onEvent.getPointerCount()(returns(the(number(of(fingers(( • Mo%onEvent.getPointerId(int)(returns(the(pointer((finger)( • Mo%onEvent’s(getX(int),(getY(int)((accept(an(index(argument(to( see(where(the(respec%ve(finger(is( DH2642 Interaction Programming DH2642 Interaction Android(design(paLerns( • Define(“standard”(ways(of(designing(interac%on(on(Android( hLps://www.google.com/design/spec/materialQdesign/ introduc%on.html( • For(consistency(and(predictability(within(and(between(apps( • And(possible(implementa%ons( • E.g.(the(“Swipe(view”(paLern(and(implementa%on( hLp://developer.android.com/training/implemen%ngQ naviga%on/lateral.html( • Most(other(graphical(plaiorms(have(design*guidelines* ( DH2642 Interaction Programming NOTIFICATIONS* DH2642 Interaction Programming DH2642 Interaction No%fica%ons( //'Let's'check'if'the'browser'supports'notifications' if'(!("Notification"'in'window))'{' ''alert(”Browser'doesn’t'support'desktop'notification");' }' ' //'Let's'check'if'the'user'is'okay'to'get'notification' else'if'(Notification.permission'==='"granted")'{' ''//'If'it's'okay'let's'create'a'notification' ''var'notification'='new'Notification("Hi'there!");' }'' else'{' ''Notification.requestPermission(function'(permission)'{' ''''//'Handle'the'permission' ''})'' }' DH2642 Interaction Programming ' ( ((( No%fica%ons( //'Build'the'notification' Notification'n''='new'Notification.Builder(this)' ''''''''.setContentTitle("New'mail'from'"'+'"[email protected]")' ''''''''.setContentText("Subject")' ''''''''.setSmallIcon(R.drawable.icon)' ''''''''.setContentIntent(pIntent)' ''''''''.setAutoCancel(true)' ''''''''//'add'extra'buttons' ''''''''.addAction(R.drawable.icon,'"Call",'pIntent)' ''''''''.addAction(R.drawable.icon,'"More",'pIntent)' ''''''''.addAction(R.drawable.icon,'"And'more",'pIntent).build();' ' //'Get'the'manager,'don’t'forget'to'ask'permission'in'the'manifest' NotificationManager'notificationManager'='' ''(NotificationManager)'getSystemService(NOTIFICATION_SERVICE);' ' //'Notify,'pass'an'id'(0),'so'you'can'cancel'the'notification'later' notificationManager.notify(0DH2642,'n);'' Interaction Programming DH2642 Interaction TOOLKITS*AND*FRAMEWORKS* DH2642 Interaction Programming Widget(Toolkit( Application Widget toolkit Windowing system, event handler Operating system Hardware DH2642 Interaction Programming DH2642 Interaction Frameworks( • Toolkits(are(like(normal(libraries( – Applica%on(runs(and(calls(the(toolkit(when(needed( • Experience(shows(that(it(is(hard(to(manage(toolkits( • If(everything(is(allowed,(applica%ons(can(easily( become(inconsistent(with(each(other( • A(framework(enforces(a(certain(way(of(working( • Examples( – Java(Swing(( – IBM/Eclipse(SWT(( – Microsoa(MFC( – Microsoa(.NET( DH2642 Interaction Programming Frameworks( Library Framework My code Framework Class library My code, made mostly of callbacks. Inversion of control (see Spring framework) Hollywood principle (we ll call you) Template method design pattern DH2642 Interaction Programming DH2642 Interaction Frameworks( • The(most(important(types(of(Java(callbacks( – (event(listener(methods(like(ac%onPerformed()(( – (paint()(method( – Other(callbacks(in(Java:(finalize(),(Observer’s( no%fy()( • Fundament:(( – we(implement(the(callbacks,(but(never(call(them.( – we(wait(for(the(framework(to(call(them.( DH2642 Interaction Programming USER*INPUT*ARCHITECTURES* DH2642 Interaction Programming DH2642 Interaction Input(strategies( • Events(are(the(current*stage*in*an*evolu=on* – Sampling( – Interrupt( – Event( • Some(input(devices(can(only(be(read(through( sampling((and(maybe(interrupts)( – Very(old,(or(recent/experimental( • Events(require(mul%tasking( DH2642 Interaction Programming Sampling(architectures( • AKA(polling( • the(applica%on(needs(to(ask(whether(there( are(changes(in(the(input( • read/write(data(and(con%nue(execu%on( • leads(oaen(to(ac%ve(loops(in(the(applica%ons( DH2642 Interaction Programming DH2642 Interaction Interrupt(based(architectures( • the(applica%on(registers(to(be(no%fied(when(a( device(state(changes,(and(it(provides(an( interrupt(handler(procedure((callback)( • the(opera%ng(system(calls(the(procedure( when(the(device(state(changes( • leads(to(complicated(asynchronous(situa%ons( DH2642 Interaction Programming EventQbased(architectures( • the(opera%ng(system(takes(care(of(the(changes(in( the(device(states( • an(event*object*(instance)*is(generated(for(every( state(change( – contains(informa%on(about(the(device(change((e.g.(the(lea( mouse(buLon(was(pressed(at(%me(T(at(the(100,(200( coordinates)( • the(applica%ons(register((subscribe)(methods(to(be( called(when(a(certain(event(occurs((callback,(see( Inversion(of(Control(later)((( DH2642 Interaction Programming DH2642 Interaction EventQbased(architectures((cont)( • aaer(its(genera%on,(the(event(is(placed(in(an( event(queue( • in(another(thread,(an(event(dispatcher(takes( the(events(from(the(queue(and(calls(the( registered((subscribed)(callback(rou%nes( DH2642 Interaction Programming Event(architecture( • The(system(places( changes(in(a(queue( Interrupt-subroutine Input device • The(framework( consumes(events(from( Callback- the(queue(regularly( list • Send(an(event(instance( Event queue with((event(details(to(all( Callback interested(((subscribed)( par%es((dispatch)( Dispatcher • Each(subscribed(party( will(register(a(callback( DH2642 Interaction Programming