How can I have one object be controlled by multiple clients?

I have a button and I am trying to send it's transform across all clients, I am calling a RPC to tell when the button was pushed so I have that covered, but I need to visually show where the button because I am using physics to push it.
It seems that I can only send the position of the button from the main client, and for some reason send next is not working in this case. Is there a way I can set 'controlled locally' for all users?

private Vector3 correctPosition = Vector3.zero;
private Quaternion correctRotation = Quaternion.identity;
void OnPhotonSerializeView(PhotonStream stream, PhotonMessageInfo info)
{
if (stream.isWriting)
{
    stream.SendNext(this.gameObject.transform.localPosition); 
    stream.SendNext(this.gameObject.transform.localRotation);
}
else 
{
    correctPosition = (Vector3)stream.ReceiveNext();
    correctRotation = (Quaternion)stream.ReceiveNext();
}
}
void FixedUpdate()
{
    CurrentDistance = Vector3.Distance(this.transform.position, InitialPosition.position);
    if (CurrentDistance > MinDistance)
    {
        Vector3 PositionDelta = InitialPosition.position - this.transform.position;
        this.Rigidbody.velocity = PositionDelta * PositionMagic * Time.fixedDeltaTime;
}
else
{ 
    this.gameObject.transform.localPosition = Vector3.zero;
}
transform.localPosition = Vector3.Lerp(transform.localPosition, correctPosition, Time.deltaTime * 5);
transform.localRotation = Quaternion.Lerp(transform.localRotation, correctRotation, Time.deltaTime * 5);
}